| |
| """Freeze a success-only showcase after the full benchmark; never generate games.""" |
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| import math |
| import re |
| import statistics |
| import sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT / 'src')) |
| from vibegame.project import parse_project, project_sha256 |
|
|
| POLICY = 'first_three_clean_by_frozen_index_v1' |
| POLICY_DECLARED_UTC = '2026-10-04T14:31:55Z' |
| DISCLOSURE = 'Selected successes — not representative' |
| FIXED_INDICES = [0, 200, 400] |
|
|
|
|
| def sha256(path): |
| digest = hashlib.sha256() |
| with path.open('rb') as handle: |
| for block in iter(lambda: handle.read(8 * 1024 * 1024), b''): |
| digest.update(block) |
| return digest.hexdigest() |
|
|
|
|
| def safe_file(root, path): |
| if not path.is_file() or not path.resolve().is_relative_to(root.resolve()): |
| raise ValueError('Showcase evidence is missing or outside this evaluation') |
| for part in (path, *path.parents): |
| if part.is_symlink(): |
| raise ValueError('Showcase evidence cannot use symlinks') |
| if part == root: |
| break |
| return path |
|
|
|
|
| def read_json(path): |
| return json.loads(path.read_text()) |
|
|
|
|
| def finite_time(value): |
| return type(value) in (float, int) and math.isfinite(value) and value >= 0 |
|
|
|
|
| def selection_for(evaluation): |
| """Recompute selection and bind all 500 result/attempt inputs before captures.""" |
| evaluation = Path(evaluation).resolve() |
| hashes = {} |
| def read(relative): |
| path = safe_file(evaluation, evaluation / relative) |
| hashes[relative] = sha256(path) |
| return read_json(path) |
| manifest, summary = read('manifest.json'), read('summary.json') |
| if (manifest.get('expected_count') != 500 or manifest.get('run_seconds') != 60 |
| or manifest.get('smoke_test') is not False or manifest.get('sample_indices') != FIXED_INDICES |
| or summary.get('status') != 'complete' or summary.get('expected_prompts') != 500 |
| or summary.get('recorded_prompts') != 500 or summary.get('infrastructure_failures') != 0 |
| or summary.get('run_seconds') != 60 or summary.get('pass_k') != 1 or summary.get('retries') != 0 |
| or summary.get('sample_indices') != FIXED_INDICES): |
| raise ValueError('Showcase requires the complete unchanged 500-prompt pass@1 benchmark') |
| expected_paths = [evaluation / 'items' / f'{index:04d}' / 'result.json' for index in range(500)] |
| if sorted((evaluation / 'items').glob('*/result.json')) != expected_paths: |
| raise ValueError('Showcase requires exactly all 500 result files') |
| results, attempts = [], [] |
| for index in range(500): |
| result = read(f'items/{index:04d}/result.json') |
| attempt = read(f'items/{index:04d}/attempt.json') |
| if (result.get('index') != index or attempt.get('index') != index |
| or any(type(result.get(key)) is not bool for key in ('compile_success', 'clean_run_60s')) |
| or result.get('infrastructure_failure') |
| or not isinstance(attempt.get('instruction'), str) or not attempt['instruction'].strip() |
| or not attempt.get('prompt_id') or result.get('prompt_id') != attempt['prompt_id'] |
| or (result['clean_run_60s'] and (not result['compile_success'] or result.get('run_seconds', 0) < 60))): |
| raise ValueError('Showcase result evidence is inconsistent or incomplete') |
| results.append(result) |
| attempts.append(attempt) |
| if len({row['instruction'] for row in attempts}) != 500 or len({row['prompt_id'] for row in attempts}) != 500: |
| raise ValueError('Showcase benchmark has duplicate prompts') |
| compiled = sum(row['compile_success'] for row in results) |
| clean_indices = [index for index, row in enumerate(results) if row['clean_run_60s']] |
| for key, count in [('compile_success', compiled), ('clean_run_60s', len(clean_indices))]: |
| if summary.get(key + '_count') != count or not math.isclose(summary.get(key + '_rate', -1), count / 500, abs_tol=1e-12): |
| raise ValueError('Showcase summary disagrees with all 500 results') |
| if len(clean_indices) < 3: |
| raise ValueError('Fewer than three clean results; three genuine showcase captures are unavailable') |
| latencies = [row['prompt_to_launch_seconds'] for row in results |
| if row['compile_success'] and finite_time(row.get('prompt_to_launch_seconds'))] |
| metrics = {'denominator': 500, 'compile_success_count': compiled, 'compile_success_rate': compiled / 500, |
| 'clean_run_60s_count': len(clean_indices), 'clean_run_60s_rate': len(clean_indices) / 500, |
| 'timed_successful_launches': len(latencies), |
| 'prompt_to_launch_median_seconds': statistics.median(latencies) if latencies else None, |
| 'under_60s_launch_count': sum(value <= 60 for value in latencies)} |
| for key in ('prompt_to_launch_median_seconds', 'under_60s_launch_count'): |
| if summary.get(key) != metrics[key]: |
| raise ValueError('Showcase timing summary disagrees with all 500 results') |
| selected = [] |
| for index in clean_indices[:3]: |
| result, attempt = results[index], attempts[index] |
| base = f'items/{index:04d}' |
| generation = read(base + '/generation.json') |
| output = generation.get('output') |
| if not isinstance(output, str) or hashlib.sha256(output.encode()).hexdigest() != generation.get('output_sha256'): |
| raise ValueError('Selected showcase generation hash differs') |
| files = parse_project(output) |
| project_hash = project_sha256(files) |
| if result.get('project_sha256') != project_hash: |
| raise ValueError('Selected showcase project differs from its original generation') |
| check_name = result.get('validation_evidence', 'check') |
| if not isinstance(check_name, str) or not re.fullmatch(r'check(?:-resume-[0-9]{3,})?', check_name): |
| raise ValueError('Unsafe showcase validation evidence pointer') |
| check = read(base + '/' + check_name + '/validation.json') |
| if (check.get('compile_success') is not True or check.get('clean_run_60s') is not True |
| or check.get('project_sha256') != project_hash or check.get('run_seconds') != 60 |
| or check.get('godot_binary_sha256') != manifest.get('runtime_sha256') |
| or check.get('runtime', {}).get('wall_seconds', 0) < 60): |
| raise ValueError('Selected showcase lacks a matching full clean-run validation') |
| for phase in ('import', 'runtime'): |
| relative = base + '/' + check_name + '/' + phase + '.log' |
| log = safe_file(evaluation, evaluation / relative) |
| hashes[relative] = sha256(log) |
| if check.get(phase, {}).get('log_sha256') != hashes[relative]: |
| raise ValueError('Selected showcase validation log hash differs') |
| selected.append({'index': index, 'prompt_id': attempt['prompt_id'], 'project_sha256': project_hash, |
| 'generation_sha256': hashes[base + '/generation.json']}) |
| return {'schema_version': 1, 'kind': 'model-success-showcase-selection', 'policy': POLICY, |
| 'policy_declared_utc': POLICY_DECLARED_UTC, |
| 'policy_timing': 'Declared after the A100 experiment began; display selection only, no benchmark changes.', |
| 'disclosure': DISCLOSURE, 'benchmark_sample_indices': FIXED_INDICES, |
| 'showcase_indices': clean_indices[:3], 'metrics': metrics, 'samples': selected, |
| 'evaluation_manifest_sha256': hashes['manifest.json'], 'summary_sha256': hashes['summary.json'], |
| 'evidence_hashes': hashes, 'repairs': 0, 'replacement_of_failed_captures': False, |
| 'timing_disclosure': '60-second edited video, not generation time. Full-batch timings, no division by batch size.'} |
|
|
|
|
| def selection_path(evaluation): |
| return Path(evaluation) / 'showcase' / 'selection.json' |
|
|
|
|
| def load_selection(evaluation, path): |
| path = Path(path) |
| if path.is_symlink(): |
| raise ValueError('Showcase selection cannot use a symlink') |
| if path.resolve() != selection_path(evaluation).resolve(): |
| raise ValueError('Showcase must use this evaluation\'s showcase/selection.json') |
| safe_file(Path(evaluation).resolve(), selection_path(Path(evaluation).resolve())) |
| expected = selection_for(evaluation) |
| if read_json(path) != expected: |
| raise ValueError('Showcase selection or its frozen inputs changed') |
| return expected |
|
|
|
|
| def ensure_selection(evaluation): |
| path = selection_path(evaluation) |
| expected = selection_for(evaluation) |
| if path.exists(): |
| load_selection(evaluation, path) |
| else: |
| path.parent.mkdir(parents=True, exist_ok=True) |
| if path.parent.is_symlink(): |
| raise ValueError('Showcase directory cannot be a symlink') |
| with path.open('x') as handle: |
| json.dump(expected, handle, sort_keys=True, indent=2) |
| handle.write('\n') |
| return path |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument('--evaluation', type=Path, required=True) |
| args = parser.parse_args() |
| path = ensure_selection(args.evaluation) |
| print(json.dumps({'selection': str(path), 'indices': read_json(path)['showcase_indices'], 'disclosure': DISCLOSURE})) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
