code/scripts/showcase.py

#!/usr/bin/env python3
"""Freeze a success-only showcase after the full benchmark; never generate games."""
from __future__ import annotations

import argparse
import hashlib
import json
import math
import re
import statistics
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'src'))
from vibegame.project import parse_project, project_sha256

POLICY = 'first_three_clean_by_frozen_index_v1'
POLICY_DECLARED_UTC = '2026-10-04T14:31:55Z'
DISCLOSURE = 'Selected successes — not representative'
FIXED_INDICES = [0, 200, 400]


def sha256(path):
    digest = hashlib.sha256()
    with path.open('rb') as handle:
        for block in iter(lambda: handle.read(8 * 1024 * 1024), b''):
            digest.update(block)
    return digest.hexdigest()


def safe_file(root, path):
    if not path.is_file() or not path.resolve().is_relative_to(root.resolve()):
        raise ValueError('Showcase evidence is missing or outside this evaluation')
    for part in (path, *path.parents):
        if part.is_symlink():
            raise ValueError('Showcase evidence cannot use symlinks')
        if part == root:
            break
    return path


def read_json(path):
    return json.loads(path.read_text())


def finite_time(value):
    return type(value) in (float, int) and math.isfinite(value) and value >= 0


def selection_for(evaluation):
    """Recompute selection and bind all 500 result/attempt inputs before captures."""
    evaluation = Path(evaluation).resolve()
    hashes = {}
    def read(relative):
        path = safe_file(evaluation, evaluation / relative)
        hashes[relative] = sha256(path)
        return read_json(path)
    manifest, summary = read('manifest.json'), read('summary.json')
    if (manifest.get('expected_count') != 500 or manifest.get('run_seconds') != 60
            or manifest.get('smoke_test') is not False or manifest.get('sample_indices') != FIXED_INDICES
            or summary.get('status') != 'complete' or summary.get('expected_prompts') != 500
            or summary.get('recorded_prompts') != 500 or summary.get('infrastructure_failures') != 0
            or summary.get('run_seconds') != 60 or summary.get('pass_k') != 1 or summary.get('retries') != 0
            or summary.get('sample_indices') != FIXED_INDICES):
        raise ValueError('Showcase requires the complete unchanged 500-prompt pass@1 benchmark')
    expected_paths = [evaluation / 'items' / f'{index:04d}' / 'result.json' for index in range(500)]
    if sorted((evaluation / 'items').glob('*/result.json')) != expected_paths:
        raise ValueError('Showcase requires exactly all 500 result files')
    results, attempts = [], []
    for index in range(500):
        result = read(f'items/{index:04d}/result.json')
        attempt = read(f'items/{index:04d}/attempt.json')
        if (result.get('index') != index or attempt.get('index') != index
                or any(type(result.get(key)) is not bool for key in ('compile_success', 'clean_run_60s'))
                or result.get('infrastructure_failure')
                or not isinstance(attempt.get('instruction'), str) or not attempt['instruction'].strip()
                or not attempt.get('prompt_id') or result.get('prompt_id') != attempt['prompt_id']
                or (result['clean_run_60s'] and (not result['compile_success'] or result.get('run_seconds', 0) < 60))):
            raise ValueError('Showcase result evidence is inconsistent or incomplete')
        results.append(result)
        attempts.append(attempt)
    if len({row['instruction'] for row in attempts}) != 500 or len({row['prompt_id'] for row in attempts}) != 500:
        raise ValueError('Showcase benchmark has duplicate prompts')
    compiled = sum(row['compile_success'] for row in results)
    clean_indices = [index for index, row in enumerate(results) if row['clean_run_60s']]
    for key, count in [('compile_success', compiled), ('clean_run_60s', len(clean_indices))]:
        if summary.get(key + '_count') != count or not math.isclose(summary.get(key + '_rate', -1), count / 500, abs_tol=1e-12):
            raise ValueError('Showcase summary disagrees with all 500 results')
    if len(clean_indices) < 3:
        raise ValueError('Fewer than three clean results; three genuine showcase captures are unavailable')
    latencies = [row['prompt_to_launch_seconds'] for row in results
                 if row['compile_success'] and finite_time(row.get('prompt_to_launch_seconds'))]
    metrics = {'denominator': 500, 'compile_success_count': compiled, 'compile_success_rate': compiled / 500,
               'clean_run_60s_count': len(clean_indices), 'clean_run_60s_rate': len(clean_indices) / 500,
               'timed_successful_launches': len(latencies),
               'prompt_to_launch_median_seconds': statistics.median(latencies) if latencies else None,
               'under_60s_launch_count': sum(value <= 60 for value in latencies)}
    for key in ('prompt_to_launch_median_seconds', 'under_60s_launch_count'):
        if summary.get(key) != metrics[key]:
            raise ValueError('Showcase timing summary disagrees with all 500 results')
    selected = []
    for index in clean_indices[:3]:
        result, attempt = results[index], attempts[index]
        base = f'items/{index:04d}'
        generation = read(base + '/generation.json')
        output = generation.get('output')
        if not isinstance(output, str) or hashlib.sha256(output.encode()).hexdigest() != generation.get('output_sha256'):
            raise ValueError('Selected showcase generation hash differs')
        files = parse_project(output)
        project_hash = project_sha256(files)
        if result.get('project_sha256') != project_hash:
            raise ValueError('Selected showcase project differs from its original generation')
        check_name = result.get('validation_evidence', 'check')
        if not isinstance(check_name, str) or not re.fullmatch(r'check(?:-resume-[0-9]{3,})?', check_name):
            raise ValueError('Unsafe showcase validation evidence pointer')
        check = read(base + '/' + check_name + '/validation.json')
        if (check.get('compile_success') is not True or check.get('clean_run_60s') is not True
                or check.get('project_sha256') != project_hash or check.get('run_seconds') != 60
                or check.get('godot_binary_sha256') != manifest.get('runtime_sha256')
                or check.get('runtime', {}).get('wall_seconds', 0) < 60):
            raise ValueError('Selected showcase lacks a matching full clean-run validation')
        for phase in ('import', 'runtime'):
            relative = base + '/' + check_name + '/' + phase + '.log'
            log = safe_file(evaluation, evaluation / relative)
            hashes[relative] = sha256(log)
            if check.get(phase, {}).get('log_sha256') != hashes[relative]:
                raise ValueError('Selected showcase validation log hash differs')
        selected.append({'index': index, 'prompt_id': attempt['prompt_id'], 'project_sha256': project_hash,
                         'generation_sha256': hashes[base + '/generation.json']})
    return {'schema_version': 1, 'kind': 'model-success-showcase-selection', 'policy': POLICY,
            'policy_declared_utc': POLICY_DECLARED_UTC,
            'policy_timing': 'Declared after the A100 experiment began; display selection only, no benchmark changes.',
            'disclosure': DISCLOSURE, 'benchmark_sample_indices': FIXED_INDICES,
            'showcase_indices': clean_indices[:3], 'metrics': metrics, 'samples': selected,
            'evaluation_manifest_sha256': hashes['manifest.json'], 'summary_sha256': hashes['summary.json'],
            'evidence_hashes': hashes, 'repairs': 0, 'replacement_of_failed_captures': False,
            'timing_disclosure': '60-second edited video, not generation time. Full-batch timings, no division by batch size.'}


def selection_path(evaluation):
    return Path(evaluation) / 'showcase' / 'selection.json'


def load_selection(evaluation, path):
    path = Path(path)
    if path.is_symlink():
        raise ValueError('Showcase selection cannot use a symlink')
    if path.resolve() != selection_path(evaluation).resolve():
        raise ValueError('Showcase must use this evaluation\'s showcase/selection.json')
    safe_file(Path(evaluation).resolve(), selection_path(Path(evaluation).resolve()))
    expected = selection_for(evaluation)
    if read_json(path) != expected:
        raise ValueError('Showcase selection or its frozen inputs changed')
    return expected


def ensure_selection(evaluation):
    path = selection_path(evaluation)
    expected = selection_for(evaluation)
    if path.exists():
        load_selection(evaluation, path)
    else:
        path.parent.mkdir(parents=True, exist_ok=True)
        if path.parent.is_symlink():
            raise ValueError('Showcase directory cannot be a symlink')
        with path.open('x') as handle:
            json.dump(expected, handle, sort_keys=True, indent=2)
            handle.write('\n')
    return path


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--evaluation', type=Path, required=True)
    args = parser.parse_args()
    path = ensure_selection(args.evaluation)
    print(json.dumps({'selection': str(path), 'indices': read_json(path)['showcase_indices'], 'disclosure': DISCLOSURE}))


if __name__ == '__main__':
    main()