FatUnc6969's picture
Publish reviewed research tools and evidence audit
0e8028d verified
Raw History Blame Contribute Delete
7.84 kB
import hashlib
import importlib
import json
import math
from pathlib import Path
import sys
from .manifests import digest
MODELS = {
'sparse_full': dict(dimensions=4096, workload='standard',
assistance='Scripted public-record notebook compressor.',
reward='Terminal +1 escape, -1 other terminal, potential shaping and step penalty.'),
'sparse_valve': dict(dimensions=4096, workload='standard',
assistance='LegalReference prefix and scripted launch. Learner controls only the final valve.',
reward='+1 escape, -1 failure.'),
'hierarchical': dict(dimensions=12, workload='compact',
assistance='LegalReference controls all actions except latch, connection bus and final valve side.',
reward='Unmodified engine reward, one on escape and zero otherwise.'),
}
def world_seeds(values):
if not isinstance(values, (list, tuple)) or not values or len(values) > 10000:
raise ValueError('world seeds must be a nonempty list of at most 10000 integers')
if any(type(value) is not int or not 0 <= value < 2**63 for value in values):
raise ValueError('world seeds must be integers between 0 and 2**63 - 1')
if len(set(values)) != len(values):
raise ValueError('world seeds must be unique')
return list(values)
def training_worlds(train, dev):
train, dev = world_seeds(train), world_seeds(dev)
if {seed // 2 for seed in train} & {seed // 2 for seed in dev}:
raise ValueError('train and dev must use disjoint paired root groups')
return train, dev
def sources(model):
if not isinstance(model, str) or model not in MODELS:
raise ValueError('unknown checkpoint model')
from borderland.replay import source_hash
names = ['borderland_eval.checkpoints', 'borderland_eval.context',
'borderland_verify.reference', 'borderland_eval.hierarchical_training'
if model == 'hierarchical' else 'borderland_eval.training']
return {'engine': source_hash(), **{
name: hashlib.sha256(Path(importlib.import_module(name).__file__).read_bytes()).hexdigest()
for name in names}}
def checkpoint(model, weights, run):
payload = dict(format='borderland-policy-v1', model=model, weights=list(weights),
weights_sha256=digest(weights), configuration=dict(MODELS[model]),
configuration_sha256=digest(MODELS[model]), sources=sources(model), run=run,
run_configuration_sha256=digest(run), frozen=True, beam_training=False)
validate(payload)
return payload
def validate(payload):
if not isinstance(payload, dict) or payload.get('format') != 'borderland-policy-v1':
raise ValueError('unsupported checkpoint format; historical bare weights need their pinned source and receipt')
model = payload.get('model')
if not isinstance(model, str) or model not in MODELS:
raise ValueError('unknown checkpoint model')
weights = payload.get('weights')
if not isinstance(weights, list) or len(weights) != MODELS[model]['dimensions'] or any(
type(value) not in (int, float) or abs(value) > sys.float_info.max or not math.isfinite(value) for value in weights):
raise ValueError('checkpoint weights have the wrong shape or nonfinite values')
if payload.get('weights_sha256') != digest(weights):
raise ValueError('checkpoint weight hash mismatch')
if payload.get('configuration') != MODELS[model] or payload.get('configuration_sha256') != digest(MODELS[model]):
raise ValueError('checkpoint configuration mismatch')
if payload.get('sources') != sources(model):
raise ValueError('checkpoint source mismatch; use the original pinned source')
run = payload.get('run')
if not isinstance(run, dict) or payload.get('run_configuration_sha256') != digest(run):
raise ValueError('checkpoint run provenance is missing')
training_worlds(run.get('train_worlds'), run.get('dev_worlds'))
if payload.get('frozen') is not True:
raise ValueError('checkpoint must be marked frozen')
if payload.get('beam_training') is not False:
raise ValueError('these checkpoints are external local policies, not Beam weights')
return payload
def load_checkpoint(path):
def unique(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError('duplicate checkpoint key')
result[key] = value
return result
with Path(path).open('rb') as source:
data = source.read(2_000_001)
if len(data) > 2_000_000:
raise ValueError('checkpoint exceeds the two-megabyte size limit')
try:
payload = json.loads(data, object_pairs_hook=unique)
except RecursionError as error:
raise ValueError('checkpoint JSON exceeds parser depth') from error
return validate(payload), hashlib.sha256(data).hexdigest()
def evaluate_checkpoint(path, *, seeds=None, split='dev', final_test=False, rng_seed=97000):
if split not in ('dev', 'fresh', 'test'):
raise ValueError('evaluation split must be dev, fresh, or test')
if (split == 'test') != bool(final_test):
raise ValueError('test evaluation requires --final-test; use it only for a declared final test')
if type(rng_seed) is not int or not 0 <= rng_seed < 2**63:
raise ValueError('action RNG seed must be between 0 and 2**63 - 1')
payload, file_hash = load_checkpoint(path)
if seeds is None:
if split != 'dev':
raise ValueError('fresh and final-test evaluations require explicit world seeds')
seeds = payload['run']['dev_worlds']
seeds = world_seeds(seeds)
groups = {seed // 2 for seed in seeds}
if groups & {seed // 2 for seed in payload['run']['train_worlds']}:
raise ValueError('evaluation overlaps a training paired root group')
if split in ('fresh', 'test') and groups & {seed // 2 for seed in payload['run']['dev_worlds']}:
raise ValueError('fresh or final-test evaluation overlaps a development paired root group')
model = payload['model']
if model == 'hierarchical':
from .hierarchical_training import SidePolicy, evaluate
policy = SidePolicy(payload['weights'])
result = evaluate(policy, seeds, rng_seed)
else:
from .training import LinearPolicy, _evaluate
policy = LinearPolicy(payload['weights'])
result = _evaluate(policy, seeds, checkpoint=model == 'sparse_valve', rng_seed=rng_seed)
after = hashlib.sha256(Path(path).read_bytes()).hexdigest()
if after != file_hash or digest(policy.weights) != payload['weights_sha256']:
raise RuntimeError('evaluation changed checkpoint or policy weights')
return dict(format='borderland-frozen-evaluation-v1', model=model, beam_training=False,
checkpoint_sha256=file_hash, checkpoint_unchanged=True,
frozen_weights_sha256=payload['weights_sha256'], sources=payload['sources'],
configuration=payload['configuration'], evaluation_split=split, python=sys.version,
action_rng_seed=rng_seed, world_seeds=seeds,
unique_paired_root_groups=len(groups), results=result,
limits=['External local policy with declared scripted assistance. No Beam calls or updates.',
'Hashes verify local integrity and compatibility, not training authorship or truth of self-reported provenance.',
'Fresh means disjoint from checkpoint train and dev roots, not a certified population sample.',
'A final-test flag records intent. It cannot prove that an operator has never inspected those worlds.'])