RealArDor/NanoJev-SimuNRGTrans / source /scripts /summarize_appo_doom.py
RealArDor's picture
download
raw
11 kB
#!/usr/bin/env python3
"""Verify and summarize the predeclared APPO Basic versus Jev experiment."""
import argparse
import json
from pathlib import Path
from summarize_unified_games import load_run_sources, summarize_run, check_cohort
from summarize_unified_cohort import cohort_view
from unified_game_pipeline import file_digest, read_rows, SPLITS
def require(condition, message):
if not condition:
raise ValueError(message)
def summarize_rendering(path, primary):
raw = json.loads(Path(path).read_text())
require(raw['exploratory'] and raw['not_part_of_primary_comparison'], 'Follow-up must remain separate')
require(raw['implementation_sha256'] == file_digest(Path(__file__).with_name('diagnose_appo_doom_rendering.py')),
'Rendering diagnostic source hash mismatch')
cases = {c['id']: c for c in read_rows(primary['protocol']['case_file']) if c['spec'].get('scenario') == 'basic'}
expected_models = [primary['protocol']['primary']['checkpoint_repository'],
*primary['protocol']['secondary_repositories']]
require(set(r['model'] for r in raw['episodes']) == set(expected_models), 'Incomplete model set')
result = {'source': str(path), 'source_sha256': file_digest(path), 'exploratory': True,
'rendering': raw['rendering'], 'controller': 'greedy', 'epsilon': .1, 'sampling_seed': 17,
'summary': {}, 'per_episode': [], 'independent_replay_passed': True,
'replayed_decisions': sum(len(r['actions']) for r in raw['episodes'])}
for model in expected_models:
rows = [r for r in raw['episodes'] if r['model'] == model]
require(len(rows) == len(cases) and {r['case']['id']: r['case'] for r in rows} == cases,
'Follow-up must cover the same 48 cases exactly once per model')
model_summary = {}
for row in rows:
require(row['independent_replay_passed'] and row['success'] == (row['metrics']['kills'] > 0),
'Missing independent replay or invalid success')
require(sum(s['actual_ticks'] for s in row['actions']) == row['metrics']['physical_ticks'],
'Physical tick sum mismatch')
result['per_episode'].append({k:v for k,v in row.items() if k != 'actions'})
for split in SPLITS:
split_rows = [r for r in rows if r['case']['split'] == split]
model_summary[split] = {'n': len(split_rows), 'successes': sum(r['success'] for r in split_rows),
'mean_metrics': {key: sum(r['metrics'][key] for r in split_rows) / len(split_rows)
for key in ('physical_ticks', 'ammo_consumed', 'native_reward')}}
result['summary'][model] = model_summary
return result
def build(protocol_path, run_paths):
protocol = json.loads(protocol_path.read_text())
cases = {c['id']: c for c in read_rows(protocol['case_file'])
if c['spec'].get('scenario') == 'basic' and c['spec']['task'] == 'shooting'}
reference = protocol['reference']
require(file_digest(reference['episodes']) == reference['sha256'], 'Jev source hash mismatch')
complete_jev = load_run_sources('Jev', reference['episodes'])
jev, selection = cohort_view(complete_jev, {'cases': cases})
require(jev['policy']['epsilon'] == .1 and jev['policy']['controller'] == 'greedy', 'Jev controller mismatch')
raw_jev = {r['case']['id']: r for r in read_rows(reference['episodes']) if r['case']['id'] in cases}
repositories = [protocol['primary']['checkpoint_repository'], *protocol['secondary_repositories']]
expected = {(repo, c['controller'], c['epsilon']) for repo in repositories
for c in protocol['controllers_for_every_repository']}
found, views, checks, primary_name = set(), [jev], {}, None
for name, path in run_paths.items():
run = load_run_sources(name, path)
policy = run['policy']
key = (policy['model'], policy['controller'], policy['epsilon'])
require(key in expected and key not in found, f'Unexpected or repeated experiment: {key}')
require(run['cases'] == cases, f'{name}: expected all original 48 Basic cases')
require(policy['sampling_seed'] == 17 and policy['actor_updated_during_evaluation'] is False,
f'{name}: sampling seed or inference contract changed')
require(policy['strict_model_state_load'] and policy['torch_weights_only'], f'{name}: unsafe or partial load')
require(policy['source_sha256']['unified_doom_env.py'] == jev['policy']['source_sha256']['unified_doom_env.py'],
f'{name}: environment adapter changed')
fields = ('vizdoom_version', 'scenario_config_sha256', 'scenario_wad_sha256', 'mode', 'buttons',
'frame_skip', 'max_steps', 'max_ticks', 'history_length', 'episode_start_tick',
'native_timeout_tick', 'counter_source', 'physical_tick_source', 'action_repeat')
comparisons = []
for row in read_rows(path):
previous = raw_jev[row['case']['id']]
a, b = row['final_info'], previous['final_info']
require(all(a[f] == b[f] for f in fields), f'{name}: environment contract mismatch')
require(row['steps'][0]['observation'] == previous['steps'][0]['observation'],
f'{name}: initial visible state mismatch')
require(row['success'] == (a['episode_metrics']['kills'] > 0), 'Success must use kill count')
comparisons.append({'id': row['case']['id'], 'split': row['case']['split'],
'appo_success': row['success'], 'jev_success': previous['success']})
paired = {}
for split in SPLITS:
rows = [r for r in comparisons if r['split'] == split]
paired[split] = {'n': len(rows),
'appo_only_success': [r['id'] for r in rows if r['appo_success'] and not r['jev_success']],
'jev_only_success': [r['id'] for r in rows if r['jev_success'] and not r['appo_success']],
'both_success': sum(r['appo_success'] and r['jev_success'] for r in rows),
'both_fail': sum(not r['appo_success'] and not r['jev_success'] for r in rows)}
checks[name] = {'environment_and_initial_state_match': True, 'environment_fields': list(fields),
'paired_by_split': paired}
main = protocol['primary']
if key == (main['checkpoint_repository'], main['controller'], main['epsilon']):
require(policy['model_revision'] == main['checkpoint_revision'] and
policy['checkpoint_sha256']['model'] == main['checkpoint_sha256'], 'Primary weights mismatch')
primary_name = name
found.add(key)
views.append(run)
require(found == expected and primary_name is not None, 'All nine predeclared runs must finish')
return {'schema_version': 'nanojev-appo-basic-comparison-v1',
'protocol': protocol, 'protocol_sha256': file_digest(protocol_path),
'cohort': check_cohort(views), 'reference_source_selection': selection,
'primary_run': primary_name, 'checks': checks,
'runs': {r['name']: summarize_run(r) for r in views},
'sources': {r['name']: r['sources'] for r in views},
'notes': ['Success is a positive KILLCOUNT delta, not positive native reward.',
'Test and OOD have six independent environment cases each; checkpoints reuse these cases.',
'APPO receives RGB and recurrent image history; Jev receives structured visible-object text.',
'This measures complete game controllers with different observation representations.',
'Only the primary and epsilon=0.1 controls match the existing Jev exploration rate.',
'No API calls, optimization, or checkpoint uploads were performed.']}
def markdown(report):
lines = ['# APPO Basic versus Jev', '',
f"Primary run, fixed before APPO evaluation: **{report['primary_run']}**.", '',
'| Controller | Train | Dev | Calibration | Test | OOD |',
'|---|---:|---:|---:|---:|---:|']
for name, run in report['runs'].items():
values = []
for split in SPLITS:
s = run['by_split_task'][f'{split}/shooting']
values.append(f"{s['successes']}/{s['n']} ({s['success_rate']:.1%})")
lines.append('| ' + ' | '.join([name, *values]) + ' |')
lines += ['', 'All rows cover the same 48 Basic cases. No controller was chosen using test outcomes.', '',
'| Controller | Test mean ticks | OOD mean ticks | Test mean ammo consumed | OOD mean ammo consumed |',
'|---|---:|---:|---:|---:|']
for name, run in report['runs'].items():
values = [run['by_split_task'][f'{split}/shooting']['metrics'][metric]['mean']
for metric in ['physical_ticks', 'ammo_consumed'] for split in ['test', 'ood']]
lines.append('| ' + ' | '.join([name, *[f'{v:.2f}' for v in values]]) + ' |')
lines += ['', *[f'- {note}' for note in report['notes']], '']
if 'rendering_followup' in report:
lines += ['## Training-rendering follow-up', '',
'Exploratory follow-up after verifying the original rendering settings on development cases.', '',
'| Checkpoint | Train | Dev | Calibration | Test | OOD |',
'|---|---:|---:|---:|---:|---:|']
for model, cells in report['rendering_followup']['summary'].items():
lines.append('| ' + ' | '.join([model, *[f"{cells[s]['successes']}/{cells[s]['n']}" for s in SPLITS]]) + ' |')
lines += ['', 'All use greedy action choice with 10% uniform exploration and the original case deadlines.', '']
return '\n'.join(lines)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--protocol', type=Path, default=Path('configs/appo_basic_v1.json'))
parser.add_argument('--run', action='append', required=True, metavar='NAME=EPISODES')
parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument('--rendering-report', type=Path)
args = parser.parse_args()
require(not args.output_dir.exists(), 'Use a fresh output directory')
paths = {}
for item in args.run:
name, path = item.split('=', 1)
require(name and name not in paths and name != 'Jev', 'Duplicate/reserved run name')
paths[name] = path
report = build(args.protocol, paths)
if args.rendering_report:
report['rendering_followup'] = summarize_rendering(args.rendering_report, report)
args.output_dir.mkdir(parents=True)
(args.output_dir / 'summary.json').write_text(json.dumps(report, indent=2, allow_nan=False) + '\n')
(args.output_dir / 'summary.md').write_text(markdown(report))
print(markdown(report))
if __name__ == '__main__':
main()

Xet Storage Details

Size:
11 kB
·
Xet hash:
15f4a5a7bc40c21448ba1246499c4e8b8c5a6e4385eac49f3c465f83eae6f936

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.