File size: 4,537 Bytes
df57b75 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """Versioned typed-decision records. Gold and provenance never enter wire payloads."""
import copy
import gzip
import hashlib
import json
import math
from pathlib import Path
def json_bytes(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8')
def digest(value):
return hashlib.sha256(json_bytes(value)).hexdigest()
def file_digest(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def read_jsonl(path):
path = Path(path)
with (gzip.open(path, 'rt') if path.suffix == '.gz' else path.open()) as stream:
return [json.loads(line) for line in stream if line.strip()]
def write_jsonl(path, records):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
raw = b''.join(json_bytes(row) + b'\n' for row in records)
raw = gzip.compress(raw, mtime=0) if path.suffix == '.gz' else raw
if path.exists() and path.read_bytes() != raw:
raise ValueError('Refuse to change an already frozen dataset: ' + str(path))
path.write_bytes(raw)
return {'path': str(path), 'sha256': file_digest(path), 'bytes': len(raw),
'records': len(records), 'questions': sum(len(r['questions']) for r in records)}
def question_keys(q):
if q['type'] == 'choice':
criteria = q.get('criteria')
if not isinstance(criteria, dict) or not 2 <= len(criteria) <= 255:
raise ValueError('Choice requires 2-255 criteria')
if any(not isinstance(k, str) or not k for k in criteria):
raise ValueError('Choice keys must be nonempty strings')
return list(criteria)
if q['type'] == 'noul':
if 'criteria' in q and set(q['criteria']) != {'false', 'true'}:
raise ValueError('Noul descriptions must cover both false and true')
return ['false', 'true']
if q['type'] == 'score':
criteria = q.get('criteria')
if not isinstance(criteria, list) or not 2 <= len(criteria) <= 255 or any(v is None for v in criteria):
raise ValueError('Score requires 2-255 non-null levels')
return [str(i) for i in range(len(criteria))]
raise ValueError('Unsupported typed question')
def gold_key(q):
label = q['label']
if q['type'] == 'noul':
if not isinstance(label, bool):
raise ValueError('Noul gold must be a boolean')
label = 'true' if label else 'false'
elif q['type'] == 'score':
if isinstance(label, bool) or not isinstance(label, int):
raise ValueError('Score gold must be an integer level')
label = str(label)
if label not in question_keys(q):
raise ValueError('Gold is absent from criteria')
return label
def api_payload(record, model=None):
if not record.get('questions'):
raise ValueError('Empty request')
qs = {}
for qid, q in record['questions'].items():
if not isinstance(qid, str) or not qid or not isinstance(q.get('instructions'), (str, dict, list)):
raise ValueError('Invalid question ID or instructions')
question_keys(q)
qs[qid] = {k: copy.deepcopy(q[k]) for k in ('type', 'instructions', 'criteria') if k in q}
result = {'state': copy.deepcopy(record['state']), 'questions': qs}
if model:
result = {'model': model, **result}
json_bytes(result)
return result
def validate_record(record):
meta = record['_meta']
for key in ('id', 'group_id', 'source'):
if not isinstance(meta.get(key), str) or not meta[key]:
raise ValueError('Missing provenance field: ' + key)
if not isinstance(meta.get('benchmark_split') or meta.get('split'), str):
raise ValueError('Missing assessment split')
for q in record['questions'].values():
gold_key(q)
api_payload(record)
return record
def normalized_distribution(raw, keys):
if not isinstance(raw, dict) or set(raw) != set(keys):
raise ValueError('Returned labels do not match requested labels')
values = [raw[k] for k in keys]
if any(isinstance(v, bool) or not isinstance(v, (float, int)) or not math.isfinite(v) or not 0 <= v <= 1 for v in values):
raise ValueError('Invalid probability')
total = sum(values)
# Official OpenRouter Decisions output rounds each probability to 2 decimals.
if total <= 0 or abs(total - 1) > max(1e-5, len(keys) * .005 + 1e-8):
raise ValueError('Probability sum is inconsistent with declared rounding')
return [v / total for v in values], total
|