"""Evaluate existing weights with the frozen native pointer readout, no training. Historical predictions are rounded to six decimals. Compare recovered raw probabilities before temperature scaling with their rounding intervals. Fast's 77 historical likelihood items are newly measured, never claimed as parity. """ import argparse import gc import hashlib import importlib.util import json import math from pathlib import Path import resource import statistics import time def read(path): p = Path(path) if p.stat().st_size > 2_000_000: raise ValueError('Metadata exceeds2MB') return json.loads(p.read_text()) def temperature(cal, question, count): if 'probabilit' in json.dumps(question).lower(): return cal['probability_route']['temperature'] return cal['per_option_count'].get(str(count), cal['global_temperature']) def invert_temperature(p, t): powers = [math.pow(v, t) for v in p] total = sum(powers) return [v / total for v in powers] def rss(): return next(int(line.split()[1]) * 1024 for line in Path('/proc/self/status').read_text().splitlines() if line.startswith('VmRSS:')) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--package', type=Path, required=True) parser.add_argument('--assets', type=Path, required=True) parser.add_argument('--tasks', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() if args.output.exists(): raise FileExistsError(args.output) started = time.monotonic(); cpu = time.process_time() end = started + 1200 guard = Path('/workspace/open-jev-watchdog.json') if guard.exists(): end = min(end, started + read(guard)['deadline'] - time.time() - 180) rows = {} for tier in ('easy', 'original', 'hard'): with (args.tasks / (tier + '.jsonl')).open() as stream: while line := stream.readline(65537): if len(line.encode()) > 65536: raise ValueError('Task too large') item = json.loads(line);item['tier'] = tier if item['id'] in rows or len(rows) >= 231: raise ValueError('Invalid task inventory') rows[item['id']] = item assert len(rows) == 231 spec = importlib.util.spec_from_file_location('submission_model', args.package / 'model.py') module = importlib.util.module_from_spec(spec);spec.loader.exec_module(module) config = module.verify() reference = read(args.package / 'reports/historical-public-reference.json') assert reference['skipped'] == [] and len(reference['predictions']) == 231 assert set(rows) == {p['id'] for p in reference['predictions']} assert not reference['probability_temperature'] expected_parity = 231 if config['historical_reference_readout'] == 'pointer' else 154 assert sum(not r['routed'] for r in reference['predictions']) == expected_parity model = module.load(args.assets) from decision_metrics import metric import torch smoke = read(args.package / 'example.json') for _ in range(8): model.decide(smoke['state'], smoke['question']) cal = read(args.package / 'calibrator.json') measured = []; flips = token_changes = rounded_mismatches = 0; worst = 0 for ref in reference['predictions']: if time.monotonic() >= end: raise RuntimeError('Preservation reserve reached') item = rows[ref['id']] question = {k: item['question'][k] for k in ('type', 'instructions', 'criteria') if k in item['question']} torch.cuda.synchronize();tick = time.perf_counter() out = model.decide(item['state'], question) torch.cuda.synchronize();seconds = time.perf_counter() - tick p = [out['probabilities'][k] for k in ref['option_keys']] prediction = max(range(len(p)), key=p.__getitem__) raw = invert_temperature(p, temperature(cal, question, len(p))) delta = None if not ref['routed']: delta = max(abs(a-b) for a, b in zip(raw, ref['p'])) worst = max(worst, delta) rounded_mismatches += int(delta > 0.00000051) flips += int(prediction != ref['prediction']) token_changes += int(out['input_tokens'] != ref['processed_tokens']) new = {k: ref[k] for k in ('id', 'type', 'tier', 'family', 'label', 'option_keys', 'is_probability')} new.update(p=p, raw_pointer_p=raw, prediction=prediction, processed_tokens=out['input_tokens'], routed=False, latency_seconds=seconds, historical_pointer_parity=not ref['routed'], raw_reference_probability_delta=delta, response_bytes=len(json.dumps(out, ensure_ascii=False).encode())) if 'gold_probs' in ref: new['gold_probs'] = ref['gold_probs'] new['total_variation'] = .5 * sum(abs(v-ref['gold_probs'].get(k, 0)) for k,v in zip(ref['option_keys'],p)) measured.append(new) for _ in range(8): model.decide(smoke['state'], smoke['question']) gc.collect();torch.cuda.synchronize() before = {'rss_bytes': rss(), 'gpu_bytes': torch.cuda.memory_allocated()} for _ in range(128): if time.monotonic() >= end: raise RuntimeError('Preservation reserve reached') model.decide(smoke['state'], smoke['question']) gc.collect();torch.cuda.synchronize() after = {'rss_bytes': rss(), 'gpu_bytes': torch.cuda.memory_allocated()} growth = {k: after[k] - before[k] for k in before} by = lambda key: {value: metric([r for r in measured if r[key] == value]) for value in sorted({r[key] for r in measured})} timing = {} for tier in ('easy', 'original', 'hard', 'all'): values = sorted(r['latency_seconds'] for r in measured if tier == 'all' or r['tier'] == tier) timing[tier] = {'n': len(values), 'p50_seconds': statistics.median(values), 'p95_seconds': values[min(len(values)-1, int(.95*len(values)))]} passed = not (flips or token_changes or rounded_mismatches) and growth['rss_bytes'] <= 64*1024**2 and growth['gpu_bytes'] <= 16*1024**2 output = {'status': 'PASSED' if passed else 'FAILED', 'checkpoint': config['checkpoint'], 'items': 231, 'skipped': [], 'predictions': measured, 'overall': metric(measured), 'by_tier': by('tier'), 'by_family': by('family'), 'by_type': by('type'), 'probabilities': 'calibrated trained-pointer softmax; no token logprobs', 'reference_pointer_items': expected_parity, 'new_pointer_items': 231-expected_parity, 'reference_argmax_changes': flips, 'reference_token_changes': token_changes, 'raw_probability_outside_rounding_interval': rounded_mismatches, 'raw_maximum_reference_delta': worst, 'raw_reference_rounding_tolerance': .00000051, 'tested_inference_manifest_sha256': hashlib.sha256((args.package/'inference-manifest.json').read_bytes()).hexdigest(), 'tested_package_manifest_sha256': hashlib.sha256((args.package/'manifest.json').read_bytes()).hexdigest(), 'timings': timing, 'timing_scope': 'encoding, forward and calibrated output; no network', 'memory_before': before, 'memory_after': after, 'memory_growth': growth, 'repeated_requests': 128, 'peak_rss_bytes': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024, 'peak_gpu_allocated_bytes': torch.cuda.max_memory_allocated(), 'gpu': torch.cuda.get_device_name(0), 'wall_seconds': time.monotonic()-started, 'cpu_seconds': time.process_time()-cpu, 'max_response_bytes': max(r['response_bytes'] for r in measured), 'new_training_tokens': 0, 'database_requests': 0, 'external_inference_requests': 0, 'official_rank': None} args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open('x') as stream:json.dump(output,stream,indent=2,allow_nan=False);stream.write('\n') print(json.dumps({k:v for k,v in output.items() if k not in ('predictions','by_family','by_type')}),flush=True) if not passed: raise RuntimeError('Submission parity or memory check failed') if __name__ == '__main__': main()