FatUnc6969's picture
Publish reviewed Borderland v8 research artifact
2719bf4 verified
Raw History Blame Contribute Delete
12.4 kB
import argparse
import hashlib
import json
from pathlib import Path
import sys
from .checkpoints import evaluate_checkpoint, world_seeds
def parse_seeds(text):
try:
return world_seeds([int(value) for value in text.split(',')])
except ValueError as error:
raise argparse.ArgumentTypeError(str(error)) from error
def smoke(seed=100000, generation_version=2, notebook_bytes=768, memory_profile='static'):
from borderland import Campaign
from borderland_verify.certificates import certify_twins
from borderland_verify.reference import public_reference
from borderland.replay import verify_replay
import tempfile
world_seeds([seed])
profile = dict(generation_version=generation_version, notebook_bytes=notebook_bytes, memory_profile=memory_profile)
campaign = Campaign.generated(seed, **profile)
transition = campaign.reset()
initial = transition.to_dict()
if Campaign.generated(seed, **profile).reset().to_dict() != initial:
raise RuntimeError('reset is not deterministic')
rejected = campaign.step({'tag': 'not_an_action', 'arguments': {}})
if not rejected.error or rejected.revision != transition.revision:
raise RuntimeError('invalid action mutated engine state')
prior = public_reference(transition)
boundary = False
engine_return = 0
phases = {int(transition.observation.phase)}
for step in range(256):
if transition.done:
break
note = prior.notebook(transition)
if note is not None:
transition = campaign.replace_notebook(note)
transition = campaign.step(prior.choose(transition))
if transition.error:
raise RuntimeError('public reference action was rejected')
engine_return += transition.reward
phases.add(int(transition.observation.phase))
if transition.context_reset:
boundary = True
if notebook_bytes and not transition.observation.to_dict()['notebook']['text']:
raise RuntimeError('notebook missing at fresh context boundary')
prior = public_reference(transition)
expected_return = 1
if generation_version == 3:
from borderland.engine import terminal_objective
expected_return = terminal_objective(transition.outcome, len(transition.observation.board['boarded']),
transition.observation.charge, len(initial['observation']['team']), initial['observation']['charge'])
if transition.outcome != 'escaped' or engine_return != expected_return or not boundary:
raise RuntimeError('reference smoke did not escape through fresh context boundary')
certificate = certify_twins(seed) if generation_version == 2 else None
if certificate is not None and not certificate['pass']:
raise RuntimeError('legacy matched checkpoint certificate failed')
with tempfile.TemporaryDirectory() as directory:
replay_path = Path(directory) / 'smoke.private.jsonl'
campaign.save_replay(replay_path, 'private')
replay = verify_replay(replay_path)
if not replay['verified'] or phases != set(range(1,8)):
raise RuntimeError('full seven-game replay smoke failed')
return dict(passed=True, scope='scripted legal-reference smoke, not learned control or Beam',
seed=seed, engine_return=engine_return, outcome=transition.outcome,
generation_version=generation_version, notebook_bytes=notebook_bytes, memory_profile=memory_profile,
phases=sorted(phases), private_replay_verified=replay['verified'],
context_reset_seen=boundary, fixed_memoryless_bound=certificate['fixed_memoryless_bound'] if certificate else None,
quality_claim='No generation-three memory necessity, compression, strategy or Beam benefit is established by this smoke.')
def verify_bundle(directory):
root = Path(directory).resolve()
manifest = json.loads((root / 'bundle-manifest.json').read_bytes())
checked = []
for row in manifest['files']:
path = (root / row['path']).resolve()
if not path.is_relative_to(root) or not path.is_file():
raise ValueError('invalid bundle file path')
if hashlib.sha256(path.read_bytes()).hexdigest() != row['sha256']:
raise ValueError('bundle hash mismatch: ' + row['path'])
checked.append(row['path'])
return dict(passed=True, checked_files=len(checked),
manifest_sha256=hashlib.sha256((root / 'bundle-manifest.json').read_bytes()).hexdigest(),
scope='File integrity against the local manifest, not publisher authentication.')
def _receipt_path(destination):
if destination is None:
return None
path = Path(destination).resolve()
for package in Path(__file__).resolve().parent.parent.glob('borderland*'):
if package.is_dir() and path.is_relative_to(package.resolve()):
raise ValueError('receipt must be outside installed packages')
return path
def _save(value, destination):
path = _receipt_path(destination)
if path is None:
return
path.parent.mkdir(parents=True, exist_ok=True)
with path.open('x') as output:
json.dump(value, output, indent=2, allow_nan=False)
output.write('\n')
def main(argv=None):
parser = argparse.ArgumentParser(description='Offline research tools. No provider or Beam training calls.')
commands = parser.add_subparsers(dest='command', required=True)
check = commands.add_parser('smoke', help='Run the public engine and matched-twin smoke.')
check.add_argument('--seed', type=int, default=100000)
check.add_argument('--generation-version', type=int, choices=(2,3), default=3)
check.add_argument('--notebook-bytes', type=int, default=768)
check.add_argument('--memory-profile', choices=('static','commissioning'), default='static')
streaming = commands.add_parser('streaming', help='Run the bounded public scripted commissioning baseline.')
streaming.add_argument('--seed', type=int, default=16)
streaming.add_argument('--archive-workload', choices=('compact', 'standard', 'extended'), default='standard')
streaming.add_argument('--call-budget', type=int, default=200)
streaming.add_argument('--notebook-bytes', type=int, default=768)
streaming.add_argument('--output', type=Path)
curves = commands.add_parser('curve-twins', help='Check the reviewed finite first-target-test curve pair.')
curves.add_argument('--seed', type=int, default=16)
curves.add_argument('--output', type=Path)
integrity = commands.add_parser('verify-bundle', help='Check downloaded file hashes.')
integrity.add_argument('directory', type=Path)
train = commands.add_parser('train', help='Train an assisted external local policy on explicit worlds.')
train.add_argument('--learner', choices=('hierarchical', 'sparse'), default='hierarchical')
train.add_argument('--train-seeds', type=parse_seeds, default=[100000, 100001])
train.add_argument('--dev-seeds', type=parse_seeds, default=[200000, 200001])
train.add_argument('--episodes', type=int, default=2)
train.add_argument('--rng-seed', type=int, default=41)
train.add_argument('--output', type=Path, required=True)
evaluate = commands.add_parser('evaluate', help='Evaluate a source-bound checkpoint without updates.')
evaluate.add_argument('checkpoint', type=Path)
evaluate.add_argument('--seeds', type=parse_seeds)
evaluate.add_argument('--split', choices=('dev', 'fresh', 'test'), default='dev')
evaluate.add_argument('--final-test', action='store_true')
evaluate.add_argument('--rng-seed', type=int, default=97000)
evaluate.add_argument('--output', type=Path)
offline = commands.add_parser('offline', help='Compare six scripted baselines and checkpoint certificates.')
offline.add_argument('--seeds', type=parse_seeds, default=[100000, 100001])
offline.add_argument('--generation-version', type=int, choices=(2, 3), default=2)
offline.add_argument('--archive-workload', choices=('compact', 'standard', 'extended'), default='standard')
offline.add_argument('--notebook-bytes', type=int, default=768)
offline.add_argument('--memory-profile', choices=('static', 'commissioning'), default='static')
offline.add_argument('--output', type=Path)
args = parser.parse_args(argv)
try:
if args.command == 'smoke':
result = smoke(args.seed, args.generation_version, args.notebook_bytes, args.memory_profile)
elif args.command == 'curve-twins':
path = _receipt_path(args.output)
if args.output is not None:
if args.output.exists() or args.output.is_symlink():
raise ValueError('curve-twins output must be a new file')
ancestor = path.parent
while not ancestor.exists():
ancestor = ancestor.parent
if not ancestor.is_dir():
raise ValueError('curve-twins output parent must be a directory')
from borderland_verify.curve_twins import certify_curve_twins
result = certify_curve_twins(args.seed)
_save(result, args.output)
result = result['summary']
print(json.dumps(result, indent=2, allow_nan=False))
return 0 if result['passed'] else 1
elif args.command == 'streaming':
from .streaming import run_streaming
_receipt_path(args.output)
if args.output is not None and (args.output.exists() or args.output.is_symlink()):
raise ValueError('streaming output must be a new file to preserve earlier runs')
result = run_streaming(args.seed, args.archive_workload, args.call_budget, args.notebook_bytes)
_save(result, args.output)
result = result['summary']
elif args.command == 'verify-bundle':
result = verify_bundle(args.directory)
elif args.command == 'train':
if args.output.exists() or args.output.is_symlink():
raise ValueError('training output must be a new directory to preserve earlier runs')
if args.learner == 'hierarchical':
from .hierarchical_training import train_hierarchical as trainer
else:
from .training import train_local as trainer
result = trainer(args.output, episodes=args.episodes, seed=args.rng_seed,
train_worlds=args.train_seeds, dev_worlds=args.dev_seeds)
result = {key: result[key] for key in ('learner', 'rl_updates', 'initial_checksum', 'final_checksum', 'limits')}
result.update(beam_training=False, output=str(args.output), evaluation_split='dev')
elif args.command == 'evaluate':
result = evaluate_checkpoint(args.checkpoint, seeds=args.seeds, split=args.split,
final_test=args.final_test, rng_seed=args.rng_seed)
_save(result, args.output)
else:
from .suite import run_suite
result = run_suite(args.seeds, split_manifests={}, generation_version=args.generation_version,
archive_workload=args.archive_workload, notebook_bytes=args.notebook_bytes,
memory_profile=args.memory_profile)
_save(result, args.output)
result = dict(core_pass=result['core_pass'], configuration=result['configuration'],
public_reference_pass=result['public_reference_pass'],
certificate_status=result['certificate_status'],
privileged_reference_status=result['privileged_reference']['status'],
policies=[dict(name=row['name'], scope=row['scope'], escapes=row['escapes'],
denominator=row['denominator'], dependency_metrics=row['dependency_metrics']) for row in result['policies']],
limitations=result['limitations'])
print(json.dumps(result, indent=2, allow_nan=False))
return 0
except (OSError, ValueError, RuntimeError, ImportError) as error:
print(f'{args.command} failed: {error}', file=sys.stderr)
return 1
if __name__ == '__main__':
raise SystemExit(main())