File size: 4,348 Bytes
e1acc15 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | """Fail-closed source-family/ID validation before any replay tensors are read.
Task labels may recur in train/dev; a source family is a connected component of
source documents used together. Such a component must belong to one split only.
"""
from __future__ import annotations
from collections import Counter
from collections.abc import Mapping, Sequence
from typing import cast
def rows(value: object) -> list[Mapping[str, object]]:
if not isinstance(value, list) or not all(isinstance(x, dict) for x in value):
raise ValueError('expected a list of metadata objects')
return cast(list[Mapping[str, object]], value)
def field(row: Mapping[str, object], name: str) -> str:
value = row.get(name)
if not isinstance(value, str) or not value:
raise ValueError(f'missing/non-string {name}')
return value
def source_units(row: Mapping[str, object]) -> list[str]:
value = row.get('source_docs')
if not isinstance(value, list) or not value or not all(isinstance(x, str) and x for x in value):
raise ValueError('source_docs must contain source-family identities')
return cast(list[str], value)
def validate_manifest(manifest: Mapping[str, object]) -> dict[str, Mapping[str, object]]:
prompts = rows(manifest.get('prompts'))
if manifest.get('n_prompts') != len(prompts):
raise ValueError('manifest n_prompts disagrees with exact prompt rows')
by_id: dict[str, Mapping[str, object]] = {}
hashes: set[str] = set()
units: dict[str, str] = {}
actual: dict[str, Counter[str]] = {}
heldout = {'chronology', 'motion_outline'}
for p in prompts:
pid, split, family, sha = (field(p, k) for k in ('pid', 'split', 'family', 'prompt_sha256'))
if pid in by_id or sha in hashes:
raise ValueError('duplicate prompt ID or identity')
by_id[pid] = p
hashes.add(sha)
if (family in heldout) != (split == 'heldout_legal'):
raise ValueError('held-out task family crosses split boundary')
for unit in source_units(p):
if units.setdefault(unit, split) != split:
raise ValueError('source family crosses split boundary')
actual.setdefault(split, Counter())[family] += 1
if manifest.get('counts') != {sp: dict(c) for sp, c in actual.items()}:
raise ValueError('manifest counts disagree with exact split/family rows')
excluded = manifest.get('excluded_pids', {})
if not isinstance(excluded, dict) or set(by_id) & set(excluded):
raise ValueError('excluded prompt IDs remain in the manifest')
return by_id
def audit_plan(plan: Sequence[Mapping[str, object]], manifest: Mapping[str, object]) -> dict[str, object]:
by_id = validate_manifest(manifest)
seen: set[str] = set()
identities: set[str] = set()
used: set[str] = set()
splits: Counter[str] = Counter()
for e in plan:
sid, split, family, sha = (field(e, k) for k in ('seq_id', 'split', 'family', 'prompt_sha256'))
identity = field(e, 'tokens_sha256') if e.get('kind') == 'acap' else sha
if sid in seen or identity in identities:
raise ValueError('duplicate sequence ID or prompt identity in replay plan')
seen.add(sid)
identities.add(identity)
if e.get('kind') == 'acap':
if split != 'acap' or family != 'acap' or sid in by_id or sha in {
field(p, 'prompt_sha256') for p in by_id.values()}:
raise ValueError('acap identity overlaps or has a foreign split')
else:
p = by_id.get(sid)
if p is None or (split, family, sha) != tuple(field(p, k) for k in ('split', 'family', 'prompt_sha256')):
raise ValueError('replay sequence ID/split/family/hash differs from prompt manifest')
used.add(sid)
splits[split] += 1
# Partial generation is expected under the approved time cap. Retain exact
# missing IDs in the private receipt; never invent rows to meet a headline.
return {'schema': 'd2.split-audit.v1', 'manifest_prompts': len(by_id),
'planned_sequences': len(plan), 'by_split': dict(splits),
'included_prompt_ids': sorted(used), 'not_replayed_prompt_ids': sorted(set(by_id) - used),
'source_families_disjoint': True, 'exact_ids_checked': True}
|