File size: 3,490 Bytes
2ab3846 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #!/usr/bin/env python3
"""Check maintained documentation, assets and audit bytes without opening TEST data."""
from pathlib import Path
import ast,hashlib,json,re,sys
from urllib.parse import unquote
ROOT=Path(__file__).resolve().parents[1]
def check():
missing=[];historical=[];nlinks=0
files=[ROOT/'README.md']
for name in ['docs','figures','audit','models']:
files.extend(p for p in (ROOT/name).rglob('*.md') if not p.is_symlink())
for p in files:
for target in re.findall(r'\]\(([^)]+)\)',p.read_text(errors='replace')):
target=target.strip().split(' "')[0].strip('<>')
if re.match(r'^(?:https?:|mailto:|app:|#|data:)',target):continue
target=unquote(target.split('#')[0]);nlinks+=1
if target and not (p.parent/target).exists():
row={'source':p.relative_to(ROOT).as_posix(),'target':target}
if 'prompts' in p.parts and re.fullmatch(r'figures/fig[1-5]_[a-z]+\.png',target):historical.append(row)
else:missing.append(row)
manifest=json.loads((ROOT/'figures/provenance.json').read_text());bad_assets=[]
for path,digest in manifest['inputs_sha256'].items():
if hashlib.sha256((ROOT/path).read_bytes()).hexdigest()!=digest:bad_assets.append(path)
for path,digest in manifest['outputs_sha256'].items():
if hashlib.sha256((ROOT/'figures'/path).read_bytes()).hexdigest()!=digest:bad_assets.append(path)
# Only code files are inspected for imports/syntax. No execution of research entry points.
syntax=[];unresolved=[];edges=0
for folder in ['src','scripts','tests']:
for p in (ROOT/folder).rglob('*.py'):
try:tree=ast.parse(p.read_text())
except SyntaxError as e:syntax.append({'path':str(p.relative_to(ROOT)),'error':str(e)});continue
for node in ast.walk(tree):
mods=[]
if isinstance(node,ast.Import):mods=[a.name for a in node.names]
elif isinstance(node,ast.ImportFrom) and node.module and not node.level:mods=[node.module]
for mod in mods:
if mod.split('.')[0] not in ['src','scripts']:continue
path=ROOT.joinpath(*mod.split('.'));edges+=1
if not path.with_suffix('.py').is_file() and not path.is_dir():unresolved.append({'source':str(p.relative_to(ROOT)),'module':mod})
from scripts.check_review31_blinding import check_exported_cards
from scripts.analyze_review34_llm_judge import validate_grades
deck_text=(ROOT/'audit/human/grader.html').read_text()
deck_match=re.search(r'const cards=(\[.*?\]);let i=',deck_text,re.S)
if not deck_match:raise ValueError('Accepted blinded deck is missing')
cards=check_exported_cards(ROOT/'audit/llm_judge/package/cards',source_cards=json.loads(deck_match.group(1)))
grades=validate_grades(ROOT/'audit/llm_judge/grades.jsonl')
return {'current_local_links_checked':nlinks,'missing_current_targets':missing,'original_prompt_missing_assets_disclosed':historical,'asset_hash_failures':bad_assets,'internal_import_edges':edges,'unresolved_internal_imports':unresolved,'syntax_errors':syntax,'audit_blinding':cards,'validated_grade_records':len(grades),'test_data_opened':False,'passed':not (missing or bad_assets or syntax or unresolved)}
if __name__=='__main__':
sys.path.insert(0,str(ROOT));result=check();print(json.dumps(result,indent=2));sys.exit(0 if result['passed'] else 1)
|