| """Versioned typed-decision records. Gold and provenance never enter wire payloads.""" |
| import copy |
| import gzip |
| import hashlib |
| import json |
| import math |
| from pathlib import Path |
|
|
|
|
| def json_bytes(value): |
| return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8') |
|
|
|
|
| def digest(value): |
| return hashlib.sha256(json_bytes(value)).hexdigest() |
|
|
|
|
| def file_digest(path): |
| return hashlib.sha256(Path(path).read_bytes()).hexdigest() |
|
|
|
|
| def read_jsonl(path): |
| path = Path(path) |
| with (gzip.open(path, 'rt') if path.suffix == '.gz' else path.open()) as stream: |
| return [json.loads(line) for line in stream if line.strip()] |
|
|
|
|
| def write_jsonl(path, records): |
| path = Path(path) |
| path.parent.mkdir(parents=True, exist_ok=True) |
| raw = b''.join(json_bytes(row) + b'\n' for row in records) |
| raw = gzip.compress(raw, mtime=0) if path.suffix == '.gz' else raw |
| if path.exists() and path.read_bytes() != raw: |
| raise ValueError('Refuse to change an already frozen dataset: ' + str(path)) |
| path.write_bytes(raw) |
| return {'path': str(path), 'sha256': file_digest(path), 'bytes': len(raw), |
| 'records': len(records), 'questions': sum(len(r['questions']) for r in records)} |
|
|
|
|
| def question_keys(q): |
| if q['type'] == 'choice': |
| criteria = q.get('criteria') |
| if not isinstance(criteria, dict) or not 2 <= len(criteria) <= 255: |
| raise ValueError('Choice requires 2-255 criteria') |
| if any(not isinstance(k, str) or not k for k in criteria): |
| raise ValueError('Choice keys must be nonempty strings') |
| return list(criteria) |
| if q['type'] == 'noul': |
| if 'criteria' in q and set(q['criteria']) != {'false', 'true'}: |
| raise ValueError('Noul descriptions must cover both false and true') |
| return ['false', 'true'] |
| if q['type'] == 'score': |
| criteria = q.get('criteria') |
| if not isinstance(criteria, list) or not 2 <= len(criteria) <= 255 or any(v is None for v in criteria): |
| raise ValueError('Score requires 2-255 non-null levels') |
| return [str(i) for i in range(len(criteria))] |
| raise ValueError('Unsupported typed question') |
|
|
|
|
| def gold_key(q): |
| label = q['label'] |
| if q['type'] == 'noul': |
| if not isinstance(label, bool): |
| raise ValueError('Noul gold must be a boolean') |
| label = 'true' if label else 'false' |
| elif q['type'] == 'score': |
| if isinstance(label, bool) or not isinstance(label, int): |
| raise ValueError('Score gold must be an integer level') |
| label = str(label) |
| if label not in question_keys(q): |
| raise ValueError('Gold is absent from criteria') |
| return label |
|
|
|
|
| def api_payload(record, model=None): |
| if not record.get('questions'): |
| raise ValueError('Empty request') |
| qs = {} |
| for qid, q in record['questions'].items(): |
| if not isinstance(qid, str) or not qid or not isinstance(q.get('instructions'), (str, dict, list)): |
| raise ValueError('Invalid question ID or instructions') |
| question_keys(q) |
| qs[qid] = {k: copy.deepcopy(q[k]) for k in ('type', 'instructions', 'criteria') if k in q} |
| result = {'state': copy.deepcopy(record['state']), 'questions': qs} |
| if model: |
| result = {'model': model, **result} |
| json_bytes(result) |
| return result |
|
|
|
|
| def validate_record(record): |
| meta = record['_meta'] |
| for key in ('id', 'group_id', 'source'): |
| if not isinstance(meta.get(key), str) or not meta[key]: |
| raise ValueError('Missing provenance field: ' + key) |
| if not isinstance(meta.get('benchmark_split') or meta.get('split'), str): |
| raise ValueError('Missing assessment split') |
| for q in record['questions'].values(): |
| gold_key(q) |
| api_payload(record) |
| return record |
|
|
|
|
| def normalized_distribution(raw, keys): |
| if not isinstance(raw, dict) or set(raw) != set(keys): |
| raise ValueError('Returned labels do not match requested labels') |
| values = [raw[k] for k in keys] |
| if any(isinstance(v, bool) or not isinstance(v, (float, int)) or not math.isfinite(v) or not 0 <= v <= 1 for v in values): |
| raise ValueError('Invalid probability') |
| total = sum(values) |
| |
| if total <= 0 or abs(total - 1) > max(1e-5, len(keys) * .005 + 1e-8): |
| raise ValueError('Probability sum is inconsistent with declared rounding') |
| return [v / total for v in values], total |
|
|
