| """Evaluate existing weights with the frozen native pointer readout, no training. |
| |
| Historical predictions are rounded to six decimals. Compare recovered raw |
| probabilities before temperature scaling with their rounding intervals. Fast's |
| 77 historical likelihood items are newly measured, never claimed as parity. |
| """ |
| import argparse |
| import gc |
| import hashlib |
| import importlib.util |
| import json |
| import math |
| from pathlib import Path |
| import resource |
| import statistics |
| import time |
|
|
|
|
| def read(path): |
| p = Path(path) |
| if p.stat().st_size > 2_000_000: |
| raise ValueError('Metadata exceeds2MB') |
| return json.loads(p.read_text()) |
|
|
|
|
| def temperature(cal, question, count): |
| if 'probabilit' in json.dumps(question).lower(): |
| return cal['probability_route']['temperature'] |
| return cal['per_option_count'].get(str(count), cal['global_temperature']) |
|
|
|
|
| def invert_temperature(p, t): |
| powers = [math.pow(v, t) for v in p] |
| total = sum(powers) |
| return [v / total for v in powers] |
|
|
|
|
| def rss(): |
| return next(int(line.split()[1]) * 1024 for line in Path('/proc/self/status').read_text().splitlines() |
| if line.startswith('VmRSS:')) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument('--package', type=Path, required=True) |
| parser.add_argument('--assets', type=Path, required=True) |
| parser.add_argument('--tasks', type=Path, required=True) |
| parser.add_argument('--output', type=Path, required=True) |
| args = parser.parse_args() |
| if args.output.exists(): |
| raise FileExistsError(args.output) |
| started = time.monotonic(); cpu = time.process_time() |
| end = started + 1200 |
| guard = Path('/workspace/open-jev-watchdog.json') |
| if guard.exists(): |
| end = min(end, started + read(guard)['deadline'] - time.time() - 180) |
| rows = {} |
| for tier in ('easy', 'original', 'hard'): |
| with (args.tasks / (tier + '.jsonl')).open() as stream: |
| while line := stream.readline(65537): |
| if len(line.encode()) > 65536: |
| raise ValueError('Task too large') |
| item = json.loads(line);item['tier'] = tier |
| if item['id'] in rows or len(rows) >= 231: |
| raise ValueError('Invalid task inventory') |
| rows[item['id']] = item |
| assert len(rows) == 231 |
| spec = importlib.util.spec_from_file_location('submission_model', args.package / 'model.py') |
| module = importlib.util.module_from_spec(spec);spec.loader.exec_module(module) |
| config = module.verify() |
| reference = read(args.package / 'reports/historical-public-reference.json') |
| assert reference['skipped'] == [] and len(reference['predictions']) == 231 |
| assert set(rows) == {p['id'] for p in reference['predictions']} |
| assert not reference['probability_temperature'] |
| expected_parity = 231 if config['historical_reference_readout'] == 'pointer' else 154 |
| assert sum(not r['routed'] for r in reference['predictions']) == expected_parity |
| model = module.load(args.assets) |
| from decision_metrics import metric |
| import torch |
| smoke = read(args.package / 'example.json') |
| for _ in range(8): |
| model.decide(smoke['state'], smoke['question']) |
| cal = read(args.package / 'calibrator.json') |
| measured = []; flips = token_changes = rounded_mismatches = 0; worst = 0 |
| for ref in reference['predictions']: |
| if time.monotonic() >= end: |
| raise RuntimeError('Preservation reserve reached') |
| item = rows[ref['id']] |
| question = {k: item['question'][k] for k in ('type', 'instructions', 'criteria') if k in item['question']} |
| torch.cuda.synchronize();tick = time.perf_counter() |
| out = model.decide(item['state'], question) |
| torch.cuda.synchronize();seconds = time.perf_counter() - tick |
| p = [out['probabilities'][k] for k in ref['option_keys']] |
| prediction = max(range(len(p)), key=p.__getitem__) |
| raw = invert_temperature(p, temperature(cal, question, len(p))) |
| delta = None |
| if not ref['routed']: |
| delta = max(abs(a-b) for a, b in zip(raw, ref['p'])) |
| worst = max(worst, delta) |
| rounded_mismatches += int(delta > 0.00000051) |
| flips += int(prediction != ref['prediction']) |
| token_changes += int(out['input_tokens'] != ref['processed_tokens']) |
| new = {k: ref[k] for k in ('id', 'type', 'tier', 'family', 'label', 'option_keys', 'is_probability')} |
| new.update(p=p, raw_pointer_p=raw, prediction=prediction, processed_tokens=out['input_tokens'], |
| routed=False, latency_seconds=seconds, historical_pointer_parity=not ref['routed'], |
| raw_reference_probability_delta=delta, response_bytes=len(json.dumps(out, ensure_ascii=False).encode())) |
| if 'gold_probs' in ref: |
| new['gold_probs'] = ref['gold_probs'] |
| new['total_variation'] = .5 * sum(abs(v-ref['gold_probs'].get(k, 0)) for k,v in zip(ref['option_keys'],p)) |
| measured.append(new) |
| for _ in range(8): |
| model.decide(smoke['state'], smoke['question']) |
| gc.collect();torch.cuda.synchronize() |
| before = {'rss_bytes': rss(), 'gpu_bytes': torch.cuda.memory_allocated()} |
| for _ in range(128): |
| if time.monotonic() >= end: |
| raise RuntimeError('Preservation reserve reached') |
| model.decide(smoke['state'], smoke['question']) |
| gc.collect();torch.cuda.synchronize() |
| after = {'rss_bytes': rss(), 'gpu_bytes': torch.cuda.memory_allocated()} |
| growth = {k: after[k] - before[k] for k in before} |
| by = lambda key: {value: metric([r for r in measured if r[key] == value]) for value in sorted({r[key] for r in measured})} |
| timing = {} |
| for tier in ('easy', 'original', 'hard', 'all'): |
| values = sorted(r['latency_seconds'] for r in measured if tier == 'all' or r['tier'] == tier) |
| timing[tier] = {'n': len(values), 'p50_seconds': statistics.median(values), |
| 'p95_seconds': values[min(len(values)-1, int(.95*len(values)))]} |
| passed = not (flips or token_changes or rounded_mismatches) and growth['rss_bytes'] <= 64*1024**2 and growth['gpu_bytes'] <= 16*1024**2 |
| output = {'status': 'PASSED' if passed else 'FAILED', 'checkpoint': config['checkpoint'], |
| 'items': 231, 'skipped': [], 'predictions': measured, 'overall': metric(measured), |
| 'by_tier': by('tier'), 'by_family': by('family'), 'by_type': by('type'), |
| 'probabilities': 'calibrated trained-pointer softmax; no token logprobs', |
| 'reference_pointer_items': expected_parity, 'new_pointer_items': 231-expected_parity, |
| 'reference_argmax_changes': flips, 'reference_token_changes': token_changes, |
| 'raw_probability_outside_rounding_interval': rounded_mismatches, |
| 'raw_maximum_reference_delta': worst, 'raw_reference_rounding_tolerance': .00000051, |
| 'tested_inference_manifest_sha256': hashlib.sha256((args.package/'inference-manifest.json').read_bytes()).hexdigest(), |
| 'tested_package_manifest_sha256': hashlib.sha256((args.package/'manifest.json').read_bytes()).hexdigest(), |
| 'timings': timing, 'timing_scope': 'encoding, forward and calibrated output; no network', |
| 'memory_before': before, 'memory_after': after, 'memory_growth': growth, 'repeated_requests': 128, |
| 'peak_rss_bytes': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024, |
| 'peak_gpu_allocated_bytes': torch.cuda.max_memory_allocated(), 'gpu': torch.cuda.get_device_name(0), |
| 'wall_seconds': time.monotonic()-started, 'cpu_seconds': time.process_time()-cpu, |
| 'max_response_bytes': max(r['response_bytes'] for r in measured), |
| 'new_training_tokens': 0, 'database_requests': 0, 'external_inference_requests': 0, |
| 'official_rank': None} |
| args.output.parent.mkdir(parents=True, exist_ok=True) |
| with args.output.open('x') as stream:json.dump(output,stream,indent=2,allow_nan=False);stream.write('\n') |
| print(json.dumps({k:v for k,v in output.items() if k not in ('predictions','by_family','by_type')}),flush=True) |
| if not passed: |
| raise RuntimeError('Submission parity or memory check failed') |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
