"""Versioned typed-decision records. Gold and provenance never enter wire payloads.""" import copy import gzip import hashlib import json import math from pathlib import Path def json_bytes(value): return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8') def digest(value): return hashlib.sha256(json_bytes(value)).hexdigest() def file_digest(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def read_jsonl(path): path = Path(path) with (gzip.open(path, 'rt') if path.suffix == '.gz' else path.open()) as stream: return [json.loads(line) for line in stream if line.strip()] def write_jsonl(path, records): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) raw = b''.join(json_bytes(row) + b'\n' for row in records) raw = gzip.compress(raw, mtime=0) if path.suffix == '.gz' else raw if path.exists() and path.read_bytes() != raw: raise ValueError('Refuse to change an already frozen dataset: ' + str(path)) path.write_bytes(raw) return {'path': str(path), 'sha256': file_digest(path), 'bytes': len(raw), 'records': len(records), 'questions': sum(len(r['questions']) for r in records)} def question_keys(q): if q['type'] == 'choice': criteria = q.get('criteria') if not isinstance(criteria, dict) or not 2 <= len(criteria) <= 255: raise ValueError('Choice requires 2-255 criteria') if any(not isinstance(k, str) or not k for k in criteria): raise ValueError('Choice keys must be nonempty strings') return list(criteria) if q['type'] == 'noul': if 'criteria' in q and set(q['criteria']) != {'false', 'true'}: raise ValueError('Noul descriptions must cover both false and true') return ['false', 'true'] if q['type'] == 'score': criteria = q.get('criteria') if not isinstance(criteria, list) or not 2 <= len(criteria) <= 255 or any(v is None for v in criteria): raise ValueError('Score requires 2-255 non-null levels') return [str(i) for i in range(len(criteria))] raise ValueError('Unsupported typed question') def gold_key(q): label = q['label'] if q['type'] == 'noul': if not isinstance(label, bool): raise ValueError('Noul gold must be a boolean') label = 'true' if label else 'false' elif q['type'] == 'score': if isinstance(label, bool) or not isinstance(label, int): raise ValueError('Score gold must be an integer level') label = str(label) if label not in question_keys(q): raise ValueError('Gold is absent from criteria') return label def api_payload(record, model=None): if not record.get('questions'): raise ValueError('Empty request') qs = {} for qid, q in record['questions'].items(): if not isinstance(qid, str) or not qid or not isinstance(q.get('instructions'), (str, dict, list)): raise ValueError('Invalid question ID or instructions') question_keys(q) qs[qid] = {k: copy.deepcopy(q[k]) for k in ('type', 'instructions', 'criteria') if k in q} result = {'state': copy.deepcopy(record['state']), 'questions': qs} if model: result = {'model': model, **result} json_bytes(result) return result def validate_record(record): meta = record['_meta'] for key in ('id', 'group_id', 'source'): if not isinstance(meta.get(key), str) or not meta[key]: raise ValueError('Missing provenance field: ' + key) if not isinstance(meta.get('benchmark_split') or meta.get('split'), str): raise ValueError('Missing assessment split') for q in record['questions'].values(): gold_key(q) api_payload(record) return record def normalized_distribution(raw, keys): if not isinstance(raw, dict) or set(raw) != set(keys): raise ValueError('Returned labels do not match requested labels') values = [raw[k] for k in keys] if any(isinstance(v, bool) or not isinstance(v, (float, int)) or not math.isfinite(v) or not 0 <= v <= 1 for v in values): raise ValueError('Invalid probability') total = sum(values) # Official OpenRouter Decisions output rounds each probability to 2 decimals. if total <= 0 or abs(total - 1) > max(1e-5, len(keys) * .005 + 1e-8): raise ValueError('Probability sum is inconsistent with declared rounding') return [v / total for v in values], total