| # Independent final evidence audit |
|
|
| This archives the exact CPU audit script run after evaluation. Its SHA256 is |
| recorded in [the audit result](final-audit.json). It assumes the original local |
| `runs/`, dataset and checkpoint layout; it is an execution record, not a portable |
| release-verification CLI. Use `scripts/verify_release.py` for release downloads. |
| No new games or model inference were run by this audit. |
|
|
| ```text |
| """Independent CPU-only audit of frozen final evidence; reconstructs saved actions only.""" |
| import hashlib |
| import importlib.util |
| import json |
| import math |
| import random |
| import statistics |
| import time |
| from pathlib import Path |
| from types import SimpleNamespace |
| |
| ROOT = Path.cwd() |
| OUT = ROOT / 'runs/final-review/audit.json' |
| assert not OUT.exists(), 'Refuse to overwrite audit' |
| start = time.monotonic() |
| def read(p): |
| return json.loads(Path(p).read_text()) |
| def sha(p): |
| h = hashlib.sha256() |
| with Path(p).open('rb') as f: |
| while chunk := f.read(1048576): |
| h.update(chunk) |
| return h.hexdigest() |
| def module(name): |
| spec = importlib.util.spec_from_file_location(name, ROOT / 'scripts' / name) |
| obj = importlib.util.module_from_spec(spec) |
| spec.loader.exec_module(obj) |
| return obj |
| |
| evaluator = module('evaluate_clef.py') |
| exporter = module('export_demo.py') |
| request = read('runs/final-evaluation/request.json') |
| report_path = Path('runs/final-evaluation/report.json') |
| report = read(report_path) |
| print('Loaded final report; validating frozen identities.', flush=True) |
| players = ['base', 'base-fp32', 'trained', 'random', 'heuristic'] |
| seeds = list(range(30000, 30200)) |
| assert request['players'] == players |
| assert set(report['players']) == set(players) |
| for key, expected in [('mode', 'tournament'), ('final_test', True), ('max_pieces', 200), ('seeds', seeds)]: |
| assert report[key] == request[key] == expected, key |
| for file, expected in request['source_hashes'].items(): |
| assert sha(file) == expected, file |
| assert report['provenance']['source_hashes'] == request['source_hashes'] |
| assert report['provenance']['installed_versions'] == request['installed_versions'] |
| checkpoint = Path(request['checkpoint']) |
| checkpoint_hashes = evaluator.checkpoint_hashes(checkpoint) |
| assert checkpoint_hashes == request['checkpoint_sha256'] == report['checkpoint_sha256'] |
| selection_path = Path(request['selection_file']) |
| assert sha(selection_path) == request['selection_sha256'] |
| assert read(selection_path) == report['selection'] |
| args = SimpleNamespace(seeds=tuple(seeds), final_test=True, max_pieces=200, players=players, |
| selection_file=selection_path, checkpoint=checkpoint, max_length=4096) |
| evaluator.validate_selection(args, checkpoint_hashes) |
| evaluator.validate_neural_runtimes(report['players']) |
| episodes = report['episodes'] |
| assert len(episodes) == 1000 |
| indexed = {(e['player_id'], e['seed']): e for e in episodes} |
| assert len(indexed) == 1000 |
| assert set(indexed) == {(p, s) for p in players for s in seeds} |
| metrics = ('lines', 'score', 'pieces') |
| summaries = {} |
| episode_hashes = {} |
| def summary(values): |
| if not values: |
| return dict(count=0, mean=None, median=None, p95=None) |
| return dict(count=len(values), mean=statistics.mean(values), median=statistics.median(values), |
| p95=sorted(values)[math.ceil(.95 * len(values)) - 1]) |
| for player in players: |
| metadata = read(Path('runs/final-evaluation') / player / 'player.json') |
| assert metadata == report['players'][player] |
| rows = [] |
| for seed in seeds: |
| path = Path('runs/final-evaluation') / player / f'seed-{seed}.json' |
| episode = read(path) |
| assert episode == indexed[player, seed], str(path) |
| assert episode['player']['revision'] == metadata['revision'] |
| assert episode['player']['runtime_config'] == metadata['runtime_config'] |
| assert episode['max_pieces'] == 200 |
| exporter.validated_player(episode, 0) |
| if player in ('base', 'base-fp32', 'trained'): |
| assert all(d['probabilities'] and 0 < d['input_tokens'] <= 4096 for d in episode['decisions']) |
| episode_hashes[str(path)] = sha(path) |
| rows.append(episode) |
| primary = {m: summary([0 if e['errors'] else e['outcome'][m] for e in rows]) for m in metrics} |
| observed = {m: summary([e['outcome'][m] for e in rows]) for m in metrics} |
| failed = [e['seed'] for e in rows if e['errors']] |
| cap_count = sum(e['outcome']['cap_hit'] for e in rows) |
| events = [d for e in rows for d in e['decisions']] |
| computed = dict(episodes=200, failed_episodes=len(failed), error_rate=len(failed)/200, |
| invalid_decisions=sum(er['kind']=='invalid_decision' for e in rows for er in e['errors']), |
| cap_hit_rate=cap_count/200, failure_adjusted=primary, observed_before_error=observed, |
| latency_seconds=summary([d['decision_seconds'] for d in events]), |
| input_tokens=summary([d['input_tokens'] for d in events if d.get('input_tokens') is not None]), |
| failed_seeds=failed) |
| for comparison in ('trained_vs_base', 'trained_vs_heuristic', 'trained_vs_base_fp32', 'base_fp32_vs_base'): |
| stored = report[comparison]['players'][player] |
| for key, value in computed.items(): |
| assert stored[key] == value, (comparison, player, key) |
| summaries[player] = {**computed, 'cap_count': cap_count} |
| print(f'{player}: all 200 saved replays/observations/decisions verified; mean lines={primary["lines"]["mean"]}', flush=True) |
| comparisons = {} |
| for key, left, right in [('trained_vs_base','trained','base'), ('trained_vs_heuristic','trained','heuristic'), |
| ('trained_vs_base_fp32','trained','base-fp32'), ('base_fp32_vs_base','base-fp32','base')]: |
| stored = report[key] |
| assert stored['trained_id'] == left and stored['base_id'] == right |
| assert stored['seeds'] == seeds and stored['matches_reserved_test_pool'] is True |
| assert stored['max_pieces'] == 200 and stored['selection_performed'] is False |
| assert stored['max_error_rate'] == 0 |
| comparisons[key] = {} |
| for metric in metrics: |
| def score(p, seed): |
| e = indexed[p, seed] |
| return 0 if e['errors'] else e['outcome'][metric] |
| deltas = [score(left,s)-score(right,s) for s in seeds] |
| rng = random.Random(2026) |
| means = sorted(sum(rng.choice(deltas) for _ in seeds)/200 for _ in range(10000)) |
| def percentile(q): |
| n = 9999*q |
| low = int(n) |
| return means[low] + (means[math.ceil(n)]-means[low])*(n-low) |
| expected = dict(episodes=200, mean_difference=statistics.mean(deltas), ci95_lower=percentile(.025), |
| ci95_upper=percentile(.975), method='paired episode percentile bootstrap', |
| bootstrap_samples=10000, bootstrap_seed=2026) |
| assert expected == stored['paired_trained_minus_base'][metric], (key, metric) |
| comparisons[key][metric] = expected |
| acceptable = not summaries[left]['failed_seeds'] and not summaries[right]['failed_seeds'] |
| assert stored['errors_acceptable'] == acceptable |
| assert stored['positive_mean_lines_signal'] == (comparisons[key]['lines']['ci95_lower'] > 0 and acceptable) |
| print(f'{key}: all three paired metrics and bootstrap intervals independently match.', flush=True) |
| result = dict(status='passed', audit_script_sha256=sha(__file__), elapsed_seconds=time.monotonic()-start, |
| report_sha256=sha(report_path), request_sha256=sha('runs/final-evaluation/request.json'), |
| selection_sha256=sha(selection_path), checkpoint_sha256=checkpoint_hashes, |
| source_hashes=request['source_hashes'], episode_file_sha256=episode_hashes, |
| episodes_verified=1000, summaries=summaries, comparisons=comparisons, |
| method='Saved replay reconstruction; independent statistics and random.choice paired bootstrap. No new policy decisions/model loads.') |
| OUT.write_text(json.dumps(result, indent=2, allow_nan=False)+'\n') |
| print(json.dumps({k: result[k] for k in ('status','elapsed_seconds','report_sha256','selection_sha256','episodes_verified')}), flush=True) |
| |
| ``` |
|
|
