import argparse import hashlib import json from pathlib import Path import sys from .checkpoints import evaluate_checkpoint, world_seeds def parse_seeds(text): try: return world_seeds([int(value) for value in text.split(',')]) except ValueError as error: raise argparse.ArgumentTypeError(str(error)) from error def smoke(seed=100000, generation_version=2, notebook_bytes=768, memory_profile='static'): from borderland import Campaign from borderland_verify.certificates import certify_twins from borderland_verify.reference import public_reference from borderland.replay import verify_replay import tempfile world_seeds([seed]) profile = dict(generation_version=generation_version, notebook_bytes=notebook_bytes, memory_profile=memory_profile) campaign = Campaign.generated(seed, **profile) transition = campaign.reset() initial = transition.to_dict() if Campaign.generated(seed, **profile).reset().to_dict() != initial: raise RuntimeError('reset is not deterministic') rejected = campaign.step({'tag': 'not_an_action', 'arguments': {}}) if not rejected.error or rejected.revision != transition.revision: raise RuntimeError('invalid action mutated engine state') prior = public_reference(transition) boundary = False engine_return = 0 phases = {int(transition.observation.phase)} for step in range(256): if transition.done: break note = prior.notebook(transition) if note is not None: transition = campaign.replace_notebook(note) transition = campaign.step(prior.choose(transition)) if transition.error: raise RuntimeError('public reference action was rejected') engine_return += transition.reward phases.add(int(transition.observation.phase)) if transition.context_reset: boundary = True if notebook_bytes and not transition.observation.to_dict()['notebook']['text']: raise RuntimeError('notebook missing at fresh context boundary') prior = public_reference(transition) expected_return = 1 if generation_version == 3: from borderland.engine import terminal_objective expected_return = terminal_objective(transition.outcome, len(transition.observation.board['boarded']), transition.observation.charge, len(initial['observation']['team']), initial['observation']['charge']) if transition.outcome != 'escaped' or engine_return != expected_return or not boundary: raise RuntimeError('reference smoke did not escape through fresh context boundary') certificate = certify_twins(seed) if generation_version == 2 else None if certificate is not None and not certificate['pass']: raise RuntimeError('legacy matched checkpoint certificate failed') with tempfile.TemporaryDirectory() as directory: replay_path = Path(directory) / 'smoke.private.jsonl' campaign.save_replay(replay_path, 'private') replay = verify_replay(replay_path) if not replay['verified'] or phases != set(range(1,8)): raise RuntimeError('full seven-game replay smoke failed') return dict(passed=True, scope='scripted legal-reference smoke, not learned control or Beam', seed=seed, engine_return=engine_return, outcome=transition.outcome, generation_version=generation_version, notebook_bytes=notebook_bytes, memory_profile=memory_profile, phases=sorted(phases), private_replay_verified=replay['verified'], context_reset_seen=boundary, fixed_memoryless_bound=certificate['fixed_memoryless_bound'] if certificate else None, quality_claim='No generation-three memory necessity, compression, strategy or Beam benefit is established by this smoke.') def verify_bundle(directory): root = Path(directory).resolve() manifest = json.loads((root / 'bundle-manifest.json').read_bytes()) checked = [] for row in manifest['files']: path = (root / row['path']).resolve() if not path.is_relative_to(root) or not path.is_file(): raise ValueError('invalid bundle file path') if hashlib.sha256(path.read_bytes()).hexdigest() != row['sha256']: raise ValueError('bundle hash mismatch: ' + row['path']) checked.append(row['path']) return dict(passed=True, checked_files=len(checked), manifest_sha256=hashlib.sha256((root / 'bundle-manifest.json').read_bytes()).hexdigest(), scope='File integrity against the local manifest, not publisher authentication.') def _receipt_path(destination): if destination is None: return None path = Path(destination).resolve() for package in Path(__file__).resolve().parent.parent.glob('borderland*'): if package.is_dir() and path.is_relative_to(package.resolve()): raise ValueError('receipt must be outside installed packages') return path def _save(value, destination): path = _receipt_path(destination) if path is None: return path.parent.mkdir(parents=True, exist_ok=True) with path.open('x') as output: json.dump(value, output, indent=2, allow_nan=False) output.write('\n') def main(argv=None): parser = argparse.ArgumentParser(description='Offline research tools. No provider or Beam training calls.') commands = parser.add_subparsers(dest='command', required=True) check = commands.add_parser('smoke', help='Run the public engine and matched-twin smoke.') check.add_argument('--seed', type=int, default=100000) check.add_argument('--generation-version', type=int, choices=(2,3), default=3) check.add_argument('--notebook-bytes', type=int, default=768) check.add_argument('--memory-profile', choices=('static','commissioning'), default='static') streaming = commands.add_parser('streaming', help='Run the bounded public scripted commissioning baseline.') streaming.add_argument('--seed', type=int, default=16) streaming.add_argument('--archive-workload', choices=('compact', 'standard', 'extended'), default='standard') streaming.add_argument('--call-budget', type=int, default=200) streaming.add_argument('--notebook-bytes', type=int, default=768) streaming.add_argument('--output', type=Path) curves = commands.add_parser('curve-twins', help='Check the reviewed finite first-target-test curve pair.') curves.add_argument('--seed', type=int, default=16) curves.add_argument('--output', type=Path) integrity = commands.add_parser('verify-bundle', help='Check downloaded file hashes.') integrity.add_argument('directory', type=Path) train = commands.add_parser('train', help='Train an assisted external local policy on explicit worlds.') train.add_argument('--learner', choices=('hierarchical', 'sparse'), default='hierarchical') train.add_argument('--train-seeds', type=parse_seeds, default=[100000, 100001]) train.add_argument('--dev-seeds', type=parse_seeds, default=[200000, 200001]) train.add_argument('--episodes', type=int, default=2) train.add_argument('--rng-seed', type=int, default=41) train.add_argument('--output', type=Path, required=True) evaluate = commands.add_parser('evaluate', help='Evaluate a source-bound checkpoint without updates.') evaluate.add_argument('checkpoint', type=Path) evaluate.add_argument('--seeds', type=parse_seeds) evaluate.add_argument('--split', choices=('dev', 'fresh', 'test'), default='dev') evaluate.add_argument('--final-test', action='store_true') evaluate.add_argument('--rng-seed', type=int, default=97000) evaluate.add_argument('--output', type=Path) offline = commands.add_parser('offline', help='Compare six scripted baselines and checkpoint certificates.') offline.add_argument('--seeds', type=parse_seeds, default=[100000, 100001]) offline.add_argument('--generation-version', type=int, choices=(2, 3), default=2) offline.add_argument('--archive-workload', choices=('compact', 'standard', 'extended'), default='standard') offline.add_argument('--notebook-bytes', type=int, default=768) offline.add_argument('--memory-profile', choices=('static', 'commissioning'), default='static') offline.add_argument('--output', type=Path) args = parser.parse_args(argv) try: if args.command == 'smoke': result = smoke(args.seed, args.generation_version, args.notebook_bytes, args.memory_profile) elif args.command == 'curve-twins': path = _receipt_path(args.output) if args.output is not None: if args.output.exists() or args.output.is_symlink(): raise ValueError('curve-twins output must be a new file') ancestor = path.parent while not ancestor.exists(): ancestor = ancestor.parent if not ancestor.is_dir(): raise ValueError('curve-twins output parent must be a directory') from borderland_verify.curve_twins import certify_curve_twins result = certify_curve_twins(args.seed) _save(result, args.output) result = result['summary'] print(json.dumps(result, indent=2, allow_nan=False)) return 0 if result['passed'] else 1 elif args.command == 'streaming': from .streaming import run_streaming _receipt_path(args.output) if args.output is not None and (args.output.exists() or args.output.is_symlink()): raise ValueError('streaming output must be a new file to preserve earlier runs') result = run_streaming(args.seed, args.archive_workload, args.call_budget, args.notebook_bytes) _save(result, args.output) result = result['summary'] elif args.command == 'verify-bundle': result = verify_bundle(args.directory) elif args.command == 'train': if args.output.exists() or args.output.is_symlink(): raise ValueError('training output must be a new directory to preserve earlier runs') if args.learner == 'hierarchical': from .hierarchical_training import train_hierarchical as trainer else: from .training import train_local as trainer result = trainer(args.output, episodes=args.episodes, seed=args.rng_seed, train_worlds=args.train_seeds, dev_worlds=args.dev_seeds) result = {key: result[key] for key in ('learner', 'rl_updates', 'initial_checksum', 'final_checksum', 'limits')} result.update(beam_training=False, output=str(args.output), evaluation_split='dev') elif args.command == 'evaluate': result = evaluate_checkpoint(args.checkpoint, seeds=args.seeds, split=args.split, final_test=args.final_test, rng_seed=args.rng_seed) _save(result, args.output) else: from .suite import run_suite result = run_suite(args.seeds, split_manifests={}, generation_version=args.generation_version, archive_workload=args.archive_workload, notebook_bytes=args.notebook_bytes, memory_profile=args.memory_profile) _save(result, args.output) result = dict(core_pass=result['core_pass'], configuration=result['configuration'], public_reference_pass=result['public_reference_pass'], certificate_status=result['certificate_status'], privileged_reference_status=result['privileged_reference']['status'], policies=[dict(name=row['name'], scope=row['scope'], escapes=row['escapes'], denominator=row['denominator'], dependency_metrics=row['dependency_metrics']) for row in result['policies']], limitations=result['limitations']) print(json.dumps(result, indent=2, allow_nan=False)) return 0 except (OSError, ValueError, RuntimeError, ImportError) as error: print(f'{args.command} failed: {error}', file=sys.stderr) return 1 if __name__ == '__main__': raise SystemExit(main())