Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /tools /check_evidence.py
| #!/usr/bin/env python3 | |
| """check_evidence.py — resolve the CC evidence ``impl:`` manifest (CIRISAgent#911). | |
| ``evidence/cc_impl.tsv`` is the sibling spec-map manifest the Constitution's | |
| ``check_claims.py`` resolves cross-repo ``impl:CIRISAgent#911`` pointers against | |
| (see CIRISConstitution ``constitution/EVIDENCE.md``). This is the CI gate that | |
| keeps the manifest honest: every in-repo ``path#symbol`` MUST resolve to a symbol | |
| that actually exists — a moved/renamed/deleted symbol is a build failure (a | |
| spec-regression test), so a Constitution ``impl:`` pointer can never silently rot. | |
| Manifest columns (TSV): ``decimal_id claim_id repo path#symbol status`` | |
| Resolution: | |
| - ``repo=CIRISAgent`` -> resolve ``path#symbol`` in THIS repo. The file must | |
| exist and the symbol must be defined at module level (``def``/``class``/a | |
| module-level assignment) or as ``Class.method``. A dead pointer = FAIL. | |
| - ``repo=—`` -> ``open``/``normative-only`` row, no symbol; skipped. | |
| - ``status=staged`` -> the claim is spec-ahead; its row is still resolved if | |
| it carries a real ``path#symbol`` (a staged pointer that names a symbol must | |
| still resolve), but a staged row with no path (``—``) is informational. | |
| Exit non-zero on any FAIL. | |
| """ | |
| from __future__ import annotations | |
| import ast | |
| import csv | |
| import re | |
| import os | |
| import shutil | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| MANIFEST = ROOT / "evidence" / "cc_impl.tsv" | |
| COLUMNS = ["decimal_id", "claim_id", "repo", "path#symbol", "status"] | |
| # The status vocabulary defined by CIRISConstitution constitution/EVIDENCE.md. | |
| # Validated here because an unrecognised status previously passed silently, which | |
| # meant a typo ("impl " / "implemented") degraded a row to un-checked without | |
| # failing anything — the same class of rot this gate exists to prevent. | |
| KNOWN_STATUSES = {"impl", "staged", "substrate", "normative", "open"} | |
| # `open` rows are acknowledged, tracked gaps. EVIDENCE.md gives them a `REPO#issue` | |
| # pointer rather than a `path#symbol`, so they are shape-checked, not AST-resolved. | |
| # Declaring a gap is the point: silence in this manifest reads as coverage. | |
| _ISSUE_REF = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*#\d+$") | |
| def _module_symbols(tree: ast.Module) -> tuple[set[str], dict[str, set[str]]]: | |
| """Return (top-level names, {class_name: {method names}}).""" | |
| top: set[str] = set() | |
| methods: dict[str, set[str]] = {} | |
| for node in tree.body: | |
| if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): | |
| top.add(node.name) | |
| elif isinstance(node, ast.ClassDef): | |
| top.add(node.name) | |
| methods[node.name] = {m.name for m in node.body if isinstance(m, (ast.FunctionDef, ast.AsyncFunctionDef))} | |
| elif isinstance(node, ast.Assign): | |
| for t in node.targets: | |
| if isinstance(t, ast.Name): | |
| top.add(t.id) | |
| elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): | |
| top.add(node.target.id) | |
| return top, methods | |
| def resolve(pointer: str) -> str | None: | |
| """Return an error string if ``path#symbol`` does not resolve, else None.""" | |
| if "#" not in pointer: | |
| return f"pointer has no '#symbol': {pointer!r}" | |
| path_str, symbol = pointer.split("#", 1) | |
| path = ROOT / path_str | |
| if not path.exists(): | |
| return f"file does not exist: {path_str}" | |
| try: | |
| tree = ast.parse(path.read_text(), filename=str(path)) | |
| except SyntaxError as e: # pragma: no cover - would be a broken repo | |
| return f"could not parse {path_str}: {e}" | |
| top, methods = _module_symbols(tree) | |
| if "." in symbol: | |
| cls, meth = symbol.split(".", 1) | |
| if cls not in top: | |
| return f"class not found: {path_str}#{cls}" | |
| if meth not in methods.get(cls, set()): | |
| return f"method not found: {path_str}#{symbol}" | |
| return None | |
| if symbol not in top: | |
| return f"symbol not found: {path_str}#{symbol}" | |
| return None | |
| def _issue_states(refs: set[str]) -> tuple[dict[str, str], set[str]]: | |
| """Ask GitHub whether each ``REPO#issue`` is still open. | |
| Returns ``(states, unresolved)`` — and returning the second half is the | |
| point. The first version returned only the states and swallowed every | |
| failure, so a timeout, a rate limit, or an unauthenticated `gh` produced an | |
| empty dict that the caller could not tell apart from "nothing to check". | |
| With CI presenting this as the enforcement for stale compliance gaps, a | |
| total outage of the lookup read as a pass. That is the same defect this | |
| script exists to catch, one level up: a claim (CI checked the issues) that | |
| has drifted from the code (CI checked nothing). | |
| Network-optional stays available for a developer without `gh` — the caller | |
| decides what an unresolved ref means, and only enforces it where a token | |
| was supplied. | |
| """ | |
| if not refs: | |
| return {}, set() | |
| if shutil.which("gh") is None: | |
| return {}, set(refs) | |
| states: dict[str, str] = {} | |
| unresolved: set[str] = set() | |
| for ref in sorted(refs): | |
| repo, _, num = ref.partition("#") | |
| try: | |
| out = subprocess.run( | |
| ["gh", "issue", "view", num, "--repo", f"CIRISAI/{repo}", "--json", "state", "-q", ".state"], | |
| capture_output=True, | |
| text=True, | |
| timeout=20, | |
| ) | |
| except Exception: # timeout, gh crash — record it, never silently skip | |
| unresolved.add(ref) | |
| continue | |
| if out.returncode == 0 and out.stdout.strip(): | |
| states[ref] = out.stdout.strip().upper() | |
| else: | |
| unresolved.add(ref) | |
| return states, unresolved | |
| def main() -> int: | |
| if not MANIFEST.exists(): | |
| print(f"FAIL: manifest missing: {MANIFEST}", file=sys.stderr) | |
| return 1 | |
| errors: list[str] = [] | |
| resolved = skipped = declared_open = 0 | |
| open_refs: set[str] = set() | |
| with MANIFEST.open() as f: | |
| reader = csv.reader((ln for ln in f if not ln.startswith("#")), delimiter="\t") | |
| header = next(reader, None) | |
| if header != COLUMNS: | |
| print(f"FAIL: header {header} != {COLUMNS}", file=sys.stderr) | |
| return 1 | |
| for ln, row in enumerate(reader, start=2): | |
| if not row or not any(c.strip() for c in row): | |
| continue | |
| if len(row) != len(COLUMNS): | |
| errors.append(f"L{ln}: expected {len(COLUMNS)} columns, got {len(row)}") | |
| continue | |
| _decimal, claim, repo, pointer, status = (c.strip() for c in row) | |
| if status not in KNOWN_STATUSES: | |
| errors.append(f"L{ln} [{claim}]: unknown status {status!r}; expected one of {sorted(KNOWN_STATUSES)}") | |
| continue | |
| # An `open` row declares a tracked gap and points at an issue, not a | |
| # symbol. Shape-check it so a gap cannot be "declared" against a | |
| # reference nobody can follow. | |
| if status == "open": | |
| if not _ISSUE_REF.match(pointer): | |
| errors.append( | |
| f"L{ln} [{claim}]: status=open needs a REPO#issue pointer " | |
| f"(e.g. CIRISAgent#942), got {pointer!r}" | |
| ) | |
| else: | |
| declared_open += 1 | |
| open_refs.add(pointer) | |
| continue | |
| if repo != "CIRISAgent" or pointer in ("", "—"): | |
| skipped += 1 | |
| continue | |
| err = resolve(pointer) | |
| if err: | |
| errors.append(f"L{ln} [{claim}]: {err}") | |
| else: | |
| resolved += 1 | |
| # A CLOSED ISSUE BEHIND AN `open` ROW IS A LIE IN THE OTHER DIRECTION | |
| # (CIRISAgent#1002, and NULLWORKS RC3 finding F7). | |
| # | |
| # The shape check above proves the row POINTS somewhere. It cannot tell | |
| # whether the gap it declares still exists. A row saying "this control is | |
| # open, see #942" after #942 was fixed understates the system — the same | |
| # class of defect as prose overstating it, and harder to notice because | |
| # nothing about it looks stale. | |
| # | |
| # F7's recommendation was to put compliance text under exact-pin consistency | |
| # checks so claims cannot drift BEHIND or AHEAD of code. This is the behind | |
| # half; `resolve()` above is the ahead half. | |
| states, unresolved = _issue_states(open_refs) | |
| for ref, state in sorted(states.items()): | |
| if state != "OPEN": | |
| errors.append( | |
| f"[{ref}]: row declares status=open, but the issue is {state}. " | |
| "Either the gap closed and the row should move to impl/staged, or the " | |
| "issue was closed prematurely." | |
| ) | |
| # Where a token was supplied, an UNCHECKED reference is a failure, not a | |
| # silent omission. Partial success is the subtler half: one resolvable issue | |
| # used to mask any number of unresolvable ones, so the summary said "checked" | |
| # and meant "checked some". | |
| enforcing = bool(os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")) | |
| if unresolved and enforcing: | |
| for ref in sorted(unresolved): | |
| errors.append( | |
| f"[{ref}]: declared status=open but its issue state could not be read " | |
| "(gh timed out, is unauthenticated, or the issue is inaccessible). A gate " | |
| "that cannot check must not report a pass." | |
| ) | |
| if unresolved and not enforcing: | |
| checked_note = ( | |
| f", {len(states)}/{len(open_refs)} issue states checked " | |
| f"({len(unresolved)} unreadable — no GH_TOKEN, not enforced)" | |
| ) | |
| elif open_refs: | |
| checked_note = f", {len(states)}/{len(open_refs)} issue states checked" | |
| else: | |
| checked_note = ", no declared-open rows to check" | |
| for e in errors: | |
| print(f"FAIL: {e}", file=sys.stderr) | |
| print( | |
| f"cc_impl.tsv: {resolved} in-repo pointers resolved, {declared_open} declared gaps, " | |
| f"{skipped} skipped, {len(errors)} failed{checked_note}" | |
| ) | |
| return 1 if errors else 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 10.3 kB
- Xet hash:
- 881cc8be5ff7e4327f9142076c2545026f9aaa3f67a6d5d64b4aeb273dfadf9a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.