RealArDor/NanoJev-SimuNRGTrans / source /scripts /summarize_sonic_supervision.py
RealArDor's picture
download
raw
24.3 kB
#!/usr/bin/env python3
"""Summarize the frozen Sonic mixed-SFT comparison, with no inference or network.
Complete matched case sets are mandatory. Expert collection is a separate
RGB/greedy-zero-exploration reference, never a same-controller primary run.
"""
import argparse
from collections import Counter
from datetime import datetime, timezone
from decimal import Decimal, InvalidOperation
import json
import math
from pathlib import Path
import random
from summarize_unified_games import group_stats, is_num, load_run, sha256_file
from summarize_appo_supervision import mcnemar_exact
from unified_game_pipeline import behavior_distribution, digest, policy_request, read_rows, validate_cases
from evaluate_native_qwen_shooting import ORIGINAL_WEIGHT_SHA256, validate_native_answer
WEIGHTS = {'maze': 1/3, 'snake': 1/3, 'basic': 1/6, 'predict_position': 1/6}
SPLITS = ('test', 'ood')
def require(condition, message):
if not condition:
raise ValueError(message)
def read_json(path):
return json.loads(Path(path).read_text())
def category(case):
spec = case['spec']
value = spec['scenario'] if spec['task'] == 'shooting' else spec['task']
require(value in WEIGHTS, 'Unexpected task or shooting scenario')
return value
def registry(path):
cases = read_rows(path)
require(cases, 'Case registry cannot be empty')
validate_cases(cases)
require(all(c['split'] in SPLITS for c in cases), 'Primary cases must be test/OOD only')
cells = Counter((c['split'], category(c)) for c in cases)
require(set(cells) == {(s, t) for s in SPLITS for t in WEIGHTS},
'Primary cohort must cover all four categories in both splits')
return {c['id']: c for c in cases}
def money(value):
require(not isinstance(value, bool), 'Boolean accounting amount')
try:
result = Decimal(str(value))
except InvalidOperation:
raise ValueError('Invalid accounting amount') from None
require(result.is_finite() and result >= 0, 'Negative or nonfinite accounting amount')
return result
def api_accounting(manifest, case_ids):
children = manifest.get('parallel_children')
require(isinstance(children, list) and children, 'Jev needs the completed parallel accounting manifest')
shards, journal_hashes = set(), set()
reported, reserved, successes, failures = Decimal(0), Decimal(0), 0, 0
records = []
for child in children:
shard = child['shard']
require(type(shard) is int and shard >= 0 and shard not in shards, 'Repeated or invalid journal shard')
shards.add(shard)
accounting = child['accounting']
journal_sha = accounting['journal_sha256']
require(isinstance(journal_sha, str) and len(journal_sha) == 64, 'Missing journal hash')
require(journal_sha not in journal_hashes, 'One journal appears in multiple budget shards')
journal_hashes.add(journal_sha)
cost, unknown = money(accounting['reported_cost_usd']), money(accounting['unknown_reserved_usd'])
require(money(accounting['accounted_usd']) == cost + unknown, 'Child accounting sum differs')
require(cost + unknown <= money(child['budget_usd']) + Decimal('0.000000001'),
'Child accounting exceeds its application allocation')
reported += cost
reserved += unknown
successes += accounting['succeeded_calls']
failures += accounting['failed_calls']
records.append({'shard': shard, 'manifest_sha256': child['manifest_sha256'],
'episode_sha256': child['episode_sha256'], 'cases_sha256': child['cases_sha256'],
'budget_usd': child['budget_usd'], 'accounting': accounting})
require(set(manifest.get('episode_journal_shards', {})) == set(case_ids), 'Missing episode-to-journal assignment')
require(set(manifest['episode_journal_shards'].values()) <= shards, 'Unknown episode journal shard')
require(reported == money(manifest['reported_cost_usd']) and
reserved == money(manifest['unknown_reserved_usd']), 'Merged API accounting differs from shards')
require(manifest['unknown_reservation_count'] == sum(len(c['accounting']['unknown_cost_ids']) for c in children),
'Unknown-cost reservation count differs')
require(sum((money(c['budget_usd']) for c in children), Decimal(0)) <= Decimal('3.000000001'),
'Parallel allocations exceed the frozen $3 application cap')
return {'reported_cost_usd': str(reported), 'unknown_reserved_usd': str(reserved),
'accounted_usd': str(reported+reserved), 'successful_requests': successes,
'failed_requests': failures, 'children': records,
'receipt_identity': '(journal shard, call ID)',
'scope': manifest.get('accounting_scope'),
'verification': 'Recomputed from the completed parallel manifest; not a live gateway balance or independent billing audit.'}
def verify_episode(episode, case, policy):
"""Recompute the executed seeded controller and physical-counter consistency."""
require(episode['case'] == case, 'Case definition/spec/seed differs from the frozen registry')
require(episode.get('complete') is True and type(episode.get('success')) is bool, 'Incomplete or non-Boolean outcome')
steps, final = episode['steps'], episode['final_info']
require(final.get('terminated') is True and final.get('truncated') is False and
final.get('success') is episode['success'], 'Invalid final terminal status')
require(steps, 'This registered cohort requires a nonempty trajectory')
require(final == steps[-1]['info'], 'Final info differs from the actual terminal transition')
rng = random.Random(int(digest([case['id'], policy['sampling_seed']])[:16], 16))
physical, rewards = 0, []
for index, step in enumerate(steps):
where = f"{case['id']} decision {index}"
obs, info = step['observation'], step['info']
require(obs['task'] == case['spec']['task'] and isinstance(obs['state'], str), where + ': invalid public input')
require(type(obs['remaining_steps']) is int and obs['remaining_steps'] > 0, where + ': exhausted live deadline')
candidates, scores = obs['candidates'], step['scores']
require(1 <= len(candidates) <= 4 and set(scores) == set(candidates), where + ': action support differs')
require(all(is_num(x) and 0 <= x <= 1 for x in scores.values()) and
abs(math.fsum(scores.values())-1) <= 1e-6, where + ': malformed Choice probabilities')
forced = len(candidates) == 1
require(bool(step.get('forced', False)) == forced, where + ': forced-action declaration differs')
if forced:
require(step['answers'] == {} and next(iter(scores.values())) == 1., where + ': forced inference receipt differs')
else:
require(step['answers']['action']['probabilities'] == scores, where + ': model probabilities differ from scores')
if policy['engine'] == 'native_qwen_original_lm':
validate_native_answer(step['answers']['action'], policy_request(obs, case['id']))
if 'request' in step:
require(step['request'] == policy_request(obs, case['id']), where + ': full request differs')
expected = behavior_distribution(scores, 'greedy', .1)
require(set(step['behavior_probs']) == set(expected) and all(
is_num(step['behavior_probs'][k]) and abs(step['behavior_probs'][k]-v) <= 1e-12
for k, v in expected.items()), where + ': epsilon behavior differs')
draw, cumulative, action = rng.random(), 0., sorted(expected)[-1]
for key in sorted(expected):
cumulative += expected[key]
if draw < cumulative:
action = key
break
require(step['action'] == action, where + ': executed action differs from seeded behavior')
if 'sampling_draw' in step:
require(step['sampling_draw'] == draw, where + ': RNG receipt differs')
require(step['truncated'] is False and step['terminated'] is (index == len(steps)-1), where + ': terminal chain differs')
require(info['terminated'] is step['terminated'] and info['truncated'] is False, where + ': transition flags differ')
reward = step['reward']
require(is_num(reward), where + ': nonfinite reward')
rewards.append(reward)
metric = info['episode_metrics']
if obs['task'] == 'shooting':
duration = info['actual_ticks']
require(type(duration) is int and 1 <= duration <= info['requested_ticks'] <= case['spec']['frame_skip'], where + ': invalid physical ticks')
physical += duration
require(obs['step'] == index and metric['decisions'] == index+1 and metric['physical_ticks'] == physical,
where + ': physical/decision clock differs')
require(info['action_id'] == action, where + ': physical action differs')
else:
duration = info['physical_steps_this_action']
require(type(duration) is int and duration >= 1, where + ': invalid physical movement count')
require(obs['step'] == physical, where + ': pre-action physical clock differs')
physical += duration
require(metric['physical_steps'] == physical and metric['decision_steps'] == index+1,
where + ': macro/decision counter differs')
require(physical <= case['spec']['max_steps'], where + ': task deadline exceeded')
metric = final['episode_metrics']
require(metric['success'] is episode['success'], 'Final metric success differs')
if case['spec']['task'] == 'shooting':
require(is_num(metric['kills']) and (metric['kills'] > 0) is episode['success'], 'Doom success differs from actual kills')
require(abs(math.fsum(rewards)-metric['native_reward']) <= 1e-9 and metric['ammo_consumed'] >= 0,
'Doom reward/ammo accounting differs')
elif case['spec']['task'] == 'snake':
expected_success = metric['food_collected'] >= metric['target_food'] and metric['collisions'] == 0
require(expected_success is episode['success'], 'Snake food/collision success differs')
else:
require((metric['outcome'] == 'goal_reached') is episode['success'], 'Maze goal outcome differs')
if 'final_observation' in episode:
require(episode['final_observation']['candidates'] == {}, 'Terminal observation still offers actions')
return len(steps)
def load_primary(name, path, cases, case_sha, engine, checkpoint_sha=None):
run = load_run(name, path)
require(run['cases_sha256'] == case_sha, name + ': original complete case-file hash differs')
require(set(run['cases']) == set(cases), name + ': primary cohort missing or extra cases')
policy = run['policy']
require(digest(policy) == run['continuation_policy_id'], name + ': policy identity hash differs')
expected = {'engine': engine, 'controller': 'greedy', 'epsilon': .1, 'sampling_seed': 17,
'temperature': 1., 'tie_break': 'lexicographic_first', 'environment_contract': 'finite_task_deadline_v1'}
require(all(policy.get(k) == v for k, v in expected.items()), name + ': primary controller/engine contract differs')
if engine == 'checkpoint':
require(isinstance(policy.get('checkpoint_sha256'), dict) and
policy['checkpoint_sha256'].get('best.safetensors') == checkpoint_sha,
name + ': learned checkpoint differs from selection receipt')
elif engine == 'native_qwen_original_lm':
require(policy.get('original_weight_files_sha256') == {'model.safetensors': ORIGINAL_WEIGHT_SHA256}
and policy.get('project_training_steps') == 0 and policy.get('generated_tokens') == 0,
'Untuned baseline must use the verified original vocabulary head and weights')
else:
require(policy.get('model') == 'typesafe-ai/jev', 'Jev model identity differs')
checked = 0
for episode in read_rows(path):
require(episode['case']['id'] in cases, name + ': unknown episode')
checked += verify_episode(episode, cases[episode['case']['id']], policy)
run['validation'] = {'matched_episodes': len(run['episodes']), 'controller_transitions_checked': checked,
'physical_replay_performed_by_this_script': False}
return run
def paired(reference, other):
a, b = {e['id']: e for e in reference}, {e['id']: e for e in other}
require(a and a.keys() == b.keys(), 'Paired comparison requires identical case IDs')
wins = sorted(k for k in a if b[k]['success'] and not a[k]['success'])
losses = sorted(k for k in a if a[k]['success'] and not b[k]['success'])
both = sum(a[k]['success'] and b[k]['success'] for k in a)
return {'n': len(a), 'wins_vs_jev': len(wins), 'losses_vs_jev': len(losses),
'both_success': both, 'both_failure': len(a)-len(wins)-len(losses)-both,
'success_rate_delta_vs_jev': (len(wins)-len(losses))/len(a),
'exact_mcnemar_two_sided_p': mcnemar_exact(len(wins), len(losses)),
'p_adjustment': 'none; descriptive planned comparisons',
'win_case_ids': wins, 'loss_case_ids': losses}
def statistics(run, cases):
cells = {f'{s}/{t}': group_stats([e for e in run['episodes'] if e['split'] == s and category(cases[e['id']]) == t])
for s in SPLITS for t in WEIGHTS}
return {'by_split_category': cells, 'macro_by_split': {
s: {'success_rate': math.fsum(WEIGHTS[t]*cells[f'{s}/{t}']['success_rate'] for t in WEIGHTS),
'weights': WEIGHTS, 'interval': None} for s in SPLITS}}
def expert_reference(path, protocol_path):
from prepare_sonic_supervision import load_protocol, validate_collection, validate_episode
protocol, cases, protocol_sha = load_protocol(protocol_path)
run = load_run('visual_expert_reference', path)
manifest = read_json(Path(path).with_suffix('.manifest.json'))
validate_collection(manifest, protocol, cases, protocol_sha)
count = 0
with Path(path).open() as handle:
for line in handle:
if not line.strip():
continue
require(count < len(cases), 'Extra expert episode')
validate_episode(json.loads(line), cases[count], manifest)
count += 1
require(count == len(cases), 'Missing expert episodes')
require(run['policy']['controller'] == 'greedy' and run['policy']['epsilon'] == 0.,
'Expected separate zero-exploration expert reference')
return {'source': provenance(run), 'episodes': count,
'by_split': {s: group_stats([e for e in run['episodes'] if e['split'] == s])
for s in protocol['case_counts']},
'comparison_scope': 'PP RGB expert collection, greedy epsilon=0. Uses old-material synchronized rendering and recurrent pixel inputs. Not a same-observation or same-controller primary comparison.'}
def provenance(run):
return {k: run[k] for k in ('episodes_path', 'manifest_path', 'episodes_sha256', 'manifest_sha256',
'cases_sha256', 'continuation_policy_id', 'policy')}
def build(args):
case_path = Path(args.cases)
cases, case_sha = registry(case_path), sha256_file(case_path)
root = Path(args.experiment)
experiment, selection = read_json(root/'experiment.json'), read_json(root/'selection.json')
protocol = experiment['protocol']
require(experiment.get('finished') is True and not experiment.get('errors'), 'Experiment is not completely finished')
require(experiment.get('test_cases_sha256') == case_sha, 'Experiment case file differs')
require(experiment.get('selection') == selection and selection.get('test_results_used') is False,
'Selection receipt differs or used heldout results')
require(protocol['population_weights'] == {
'maze/policy': 1/3, 'snake/policy': 1/3, 'shooting/policy/basic': 1/6,
'shooting/policy/predict_position': 1/6}, 'Population weighting changed')
# Check the declared dev-only selection, without using any heldout outcome.
names = ['initial'] + [a['name'] for a in protocol['arms']]
dev = selection['dev_results']
require(set(dev) == set(names), 'Missing declared dev arm')
def rank(name):
result = dev[name]
require(set(result['rates']) == set(protocol['population_weights']) and
all(is_num(v) and 0 <= v <= 1 for v in result['rates'].values()) and
is_num(result['weighted_success']), 'Invalid declared dev rates')
expected = math.fsum(protocol['population_weights'][k]*result['rates'][k] for k in protocol['population_weights'])
require(abs(expected-result['weighted_success']) <= 1e-12, 'Dev weighted score differs')
return (result['weighted_success'], result['rates']['shooting/policy/predict_position'],
result['rates']['shooting/policy/basic'], -names.index(name))
require(max(names, key=rank) == selection['selected'], 'Dev selected arm differs from the frozen rule')
require(max(names[1:], key=rank) == selection['best_new_arm'], 'Dev best-new arm differs')
selected_initial = selection['selected'] == 'initial'
initial_sha = protocol['initial_weights_sha256']
paths = {'initial': root/'initial_test.jsonl',
'selected': root/('initial_test.jsonl' if selected_initial else 'selected_test.jsonl'),
'jev': Path(args.jev), 'untuned_qwen': Path(args.native)}
shas = {'initial': initial_sha, 'selected': selection['weights_sha256']}
if selected_initial:
require(shas['selected'] == initial_sha, 'Selected initialization SHA differs')
paths['best_new'] = root/'best_new_test.jsonl'
summary = read_json(root/selection['best_new_arm']/'summary.json')
shas['best_new'] = summary['weights_sha256']
runs = {name: load_primary(name, path, cases, case_sha,
'jev' if name == 'jev' else 'native_qwen_original_lm' if name == 'untuned_qwen' else 'checkpoint',
shas.get(name)) for name, path in paths.items()}
env_fields = ('unified_grid_envs.py', 'unified_doom_env.py')
for field in env_fields:
require(all(r['policy'].get('source_sha256', {}).get(field) for r in runs.values()), 'Missing environment source hash')
require(len({r['policy']['source_sha256'][field] for r in runs.values()}) == 1,
'Primary environment source implementations differ: '+field)
summaries = {}
for name, run in runs.items():
summary = {'source': provenance(run), 'validation': run['validation'], **statistics(run, cases)}
if name != 'jev':
summary['paired_vs_jev'] = {f'{s}/{t}': paired(
[e for e in runs['jev']['episodes'] if e['split'] == s and category(cases[e['id']]) == t],
[e for e in run['episodes'] if e['split'] == s and category(cases[e['id']]) == t])
for s in SPLITS for t in WEIGHTS}
summary['macro_delta_vs_jev'] = {s: math.fsum(
WEIGHTS[t]*summary['paired_vs_jev'][f'{s}/{t}']['success_rate_delta_vs_jev'] for t in WEIGHTS)
for s in SPLITS}
summaries[name] = summary
return {'schema_version': 'nanojev-sonic-supervision-summary-v1', 'passed': True,
'created_at': datetime.now(timezone.utc).isoformat(),
'cases': {'path': str(case_path), 'sha256': case_sha, 'episodes': len(cases),
'counts': dict(Counter(f"{c['split']}/{category(c)}" for c in cases.values()))},
'experiment': {'path': str(root/'experiment.json'), 'sha256': sha256_file(root/'experiment.json'),
'selection_sha256': sha256_file(root/'selection.json'), 'selection': selection,
'dev_selection_note': 'Rule recomputed from recorded dev scores; no heldout outcomes used. This script does not rerun dev models.'},
'primary_controller': {'controller': 'greedy', 'epsilon': .1, 'sampling_seed': 17, 'temperature': 1.},
'runs': summaries,
'api_accounting': api_accounting(read_json(Path(args.jev).with_suffix('.manifest.json')), cases),
'expert_reference': expert_reference(args.expert, args.expert_protocol),
'notes': ['All primary runs cover the exact same complete frozen cases; no result-dependent filtering.',
'Macro weights are Maze 1/3, Snake 1/3, Basic 1/6 and PP 1/6; no 548-case micro-average headline.',
'Wilson intervals describe episode-level rates; exact McNemar tests use paired episode outcomes.',
'Expert-action fit is not eventual-success calibration. No probability-calibration claim is made here.',
'Controller and recorded physical counters are verified; independent simulator replay is a separate artifact.']}
def markdown(report):
lines = ['# Sonic mixed-SFT evaluation', '',
f"Complete matched cohort: {report['cases']['episodes']} episodes. Primary controller: greedy, epsilon 0.1, seed 17.", '',
'Models are selected from dev results. Test and OOD results remain separate.', '']
labels = {'maze': 'Maze', 'snake': 'Snake', 'basic': 'Basic', 'predict_position': 'Predict Position'}
for split in SPLITS:
lines.extend([f'## {split.upper()}', '', '| Run | Maze | Snake | Basic | Predict Position | Weighted macro |',
'|---|---:|---:|---:|---:|---:|'])
for name, run in report['runs'].items():
cells = run['by_split_category']
values = [f"{cells[f'{split}/{t}']['successes']}/{cells[f'{split}/{t}']['n']}" for t in WEIGHTS]
lines.append('| '+name+' | '+' | '.join(values)+f" | {run['macro_by_split'][split]['success_rate']:.2%} |")
lines.extend(['', 'Macro: Maze 1/3, Snake 1/3, Basic 1/6, Predict Position 1/6.', '',
'| Run vs Jev | Task | Wins | Losses | Rate difference | Exact paired p |',
'|---|---|---:|---:|---:|---:|'])
for name, run in report['runs'].items():
for task in WEIGHTS if name != 'jev' else []:
pair = run['paired_vs_jev'][f'{split}/{task}']
lines.append(f"| {name} | {labels[task]} | {pair['wins_vs_jev']} | {pair['losses_vs_jev']} | {pair['success_rate_delta_vs_jev']:+.2%} | {pair['exact_mcnemar_two_sided_p']:.4g} |")
lines.append('')
expert = report['expert_reference']
lines.extend(['## Visual expert reference', '', expert['comparison_scope'], '', '| Split | PP successes |', '|---|---:|'])
for split, cell in expert['by_split'].items():
lines.append(f"| {split} | {cell['successes']}/{cell['n']} |")
fees = report['api_accounting']
lines.extend(['', '## Provenance and accounting', '',
f"Cases SHA256: `{report['cases']['sha256']}`.",
f"Selected weights SHA256: `{report['experiment']['selection']['weights_sha256']}`.",
f"API reported cost: ${fees['reported_cost_usd']}; unknown-cost reservations: ${fees['unknown_reserved_usd']}.",
fees['verification'], '', 'Detailed Wilson intervals, physical counters, source hashes and paired case IDs are in `summary.json`.', ''])
lines.extend('- '+note for note in report['notes'])
return '\n'.join(lines)+'\n'
def main():
parser = argparse.ArgumentParser(description=__doc__)
for name in ('cases', 'experiment', 'jev', 'native', 'expert', 'output'):
parser.add_argument('--'+name, type=Path, required=True)
parser.add_argument('--expert-protocol', type=Path, default=Path('configs/sonic_predict_supervision_v1.json'))
args = parser.parse_args()
require(not args.output.exists(), 'Use a fresh output directory')
report = build(args)
args.output.mkdir(parents=True)
(args.output/'summary.json').write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
(args.output/'summary.md').write_text(markdown(report))
print(json.dumps({'passed': True, 'episodes': report['cases']['episodes'], 'runs': list(report['runs']), 'output': str(args.output)}))
if __name__ == '__main__':
main()

Xet Storage Details

Size:
24.3 kB
·
Xet hash:
12421bee7f1b48ff7184da861557287ffa8eb4228905f2b1382458500976cd37

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.