File size: 4,522 Bytes
952e1e7 7027070 952e1e7 7027070 952e1e7 7027070 952e1e7 7027070 952e1e7 7027070 952e1e7 7027070 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | # SPDX-License-Identifier: MIT
"""Index-style sequential Engine benchmark, including native score question validation.
The pinned official kit only validates choice/noul. This driver retains its Engine
API and timing boundaries for the frozen mixed choice/noul/score latency sample.
"""
import argparse
import datetime
import json
import math
from pathlib import Path
import time
from probabilities import distribution
from decision_index.engines import Unsupported
from reax13_engine import Reax13Engine
def validate(questions, response):
if set(response.get('answers', {})) != set(questions):
raise ValueError('question keys mismatch')
for key, q in questions.items():
a = response['answers'][key]
if a.get('type') != q['type']:
raise ValueError('question type mismatch')
if q['type'] == 'noul':
if not math.isfinite(a['noul']) or not 0 <= a['noul'] <= 1:
raise ValueError('invalid noul probability')
else:
p = distribution(q, a)
if abs(math.fsum(p.values()) - 1) > 1e-12:
raise ValueError('not normalized')
if q['type'] == 'choice' and a['choice'] != max(sorted(p), key=p.get):
raise ValueError('choice does not match argmax')
if q['type'] == 'score' and abs(a['score'] - math.fsum(int(k)*v for k,v in p.items())) > 1e-9:
raise ValueError('score does not match expectation')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--rows', required=True)
ap.add_argument('--out', required=True)
ap.add_argument('--config', default='/engine/reax13.json')
args = ap.parse_args()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
rows = [json.loads(line) for line in Path(args.rows).read_text().splitlines()]
def status(**fields):
record = {'time': datetime.datetime.now(datetime.timezone.utc).isoformat(), **fields}
(out/'status.json').write_text(json.dumps(record))
print(json.dumps(record), flush=True)
engine = None
try:
status(event='loading')
started = time.perf_counter()
engine = Reax13Engine(config=args.config, warmup_rows=args.rows)
(out/'environment.json').write_text(json.dumps({'provenance':engine.provenance, **engine.runtime(),
'loaded_seconds':time.perf_counter()-started, 'latency':engine.latency,
'driver':'own index-style Engine API clock; adds native score validation to official choice/noul contract'},indent=2))
engine.warmup()
engine.synchronize()
status(event='ready')
start = time.perf_counter()
counts = {'ok':0, 'unsupported':0}
with (out/'results.jsonl').open('w') as f:
for i, row in enumerate(rows):
# Same clock boundaries as official runner: synchronize, request,
# synchronize, validate. Includes JSON IPC and all native work.
tick = time.perf_counter()
engine.synchronize()
try:
response, raw = engine(row['state'], row['questions'])
except Unsupported as exc:
elapsed = (time.perf_counter()-tick)*1000
f.write(json.dumps({**row['_evaluation'], 'status':'unsupported', 'error':str(exc), 'total_wall_ms':elapsed,
'payload':{'state':row['state'],'questions':row['questions']}})+'\n')
f.flush()
counts['unsupported'] += 1
continue
engine.synchronize()
validate(row['questions'], response)
elapsed = (time.perf_counter()-tick)*1000
result = {**row['_evaluation'], 'status':'ok', 'response':response,
'raw_output':raw, 'total_wall_ms':elapsed, 'model_request_wall_ms':elapsed,
'payload':{'state':row['state'],'questions':row['questions']}}
f.write(json.dumps(result)+'\n')
f.flush()
counts['ok'] += 1
if (i+1)%10 == 0:
status(event='progress',completed=i+1,counts=dict(counts),elapsed_seconds=time.perf_counter()-start)
status(event='complete',completed=len(rows),counts=dict(counts),elapsed_seconds=time.perf_counter()-start)
finally:
if engine is not None:
engine.close()
if __name__ == '__main__':
main()
|