"""JSONL dataset schema, deterministic smoke data, and split validation.""" from __future__ import annotations import hashlib import json import random from collections import Counter from collections.abc import Iterable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any @dataclass(frozen=True) class Example: id: str task_group: str state: str question: str options: tuple[str, ...] label: str | None answerable: bool split: str provenance: Mapping[str, Any] metadata: Mapping[str, Any] def to_mapping(self) -> dict[str, Any]: return { "id": self.id, "task_group": self.task_group, "state": self.state, "question": self.question, "options": list(self.options), "label": self.label, "answerable": self.answerable, "split": self.split, "provenance": dict(self.provenance), "metadata": dict(self.metadata), } @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> Example: required = ("id", "task_group", "state", "question", "options", "answerable", "split") missing = [key for key in required if key not in value] if missing: raise ValueError(f"missing dataset fields: {', '.join(missing)}") options = tuple(str(item) for item in value["options"]) label = value.get("label") if label is not None: label = str(label) if bool(value["answerable"]) and label is None: raise ValueError(f"answerable example {value['id']!r} requires a label") if not bool(value["answerable"]) and label is not None: raise ValueError(f"unanswerable example {value['id']!r} cannot have a label") if label is not None and label not in options: raise ValueError(f"label {label!r} is not in options for {value['id']!r}") return cls( id=str(value["id"]), task_group=str(value["task_group"]), state=str(value["state"]), question=str(value["question"]), options=options, label=label, answerable=bool(value["answerable"]), split=str(value["split"]), provenance=dict(value.get("provenance", {})), metadata=dict(value.get("metadata", {})), ) def validate_examples(examples: Iterable[Example]) -> None: rows = list(examples) ids = [row.id for row in rows] if len(ids) != len(set(ids)): raise ValueError("dataset example ids must be unique") allowed_splits = {"train", "development", "calibration", "test", "smoke"} scenario_splits: dict[str, str] = {} for row in rows: if row.split not in allowed_splits: raise ValueError(f"unsupported split {row.split!r} for {row.id!r}") if not 2 <= len(row.options) <= 32 or len(set(row.options)) != len(row.options): raise ValueError(f"options must contain 2..32 unique values for {row.id!r}") scenario = str(row.metadata.get("scenario_id", row.id)) previous = scenario_splits.setdefault(scenario, row.split) if previous != row.split: raise ValueError(f"scenario {scenario!r} appears in multiple splits") def read_jsonl(path: str | Path) -> list[Example]: examples: list[Example] = [] with Path(path).open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue try: examples.append(Example.from_mapping(json.loads(line))) except (json.JSONDecodeError, ValueError) as exc: raise ValueError(f"invalid JSONL at {path}:{line_number}: {exc}") from exc validate_examples(examples) return examples def write_jsonl(path: str | Path, examples: Iterable[Example]) -> None: target = Path(path) target.parent.mkdir(parents=True, exist_ok=True) with target.open("w", encoding="utf-8") as handle: for example in examples: handle.write(json.dumps(example.to_mapping(), ensure_ascii=False, sort_keys=True) + "\n") def dataset_manifest(path: str | Path, examples: Iterable[Example]) -> dict[str, Any]: rows = list(examples) encoded = "".join(json.dumps(row.to_mapping(), sort_keys=True) + "\n" for row in rows).encode() counts: dict[str, int] = {} for row in rows: counts[row.split] = counts.get(row.split, 0) + 1 labels = Counter(row.label for row in rows if row.label is not None) return { "path": str(path), "sha256": hashlib.sha256(encoded).hexdigest(), "rows": len(rows), "splits": counts, "labels": dict(sorted(labels.items())), "unanswerable_rows": sum(not row.answerable for row in rows), "task_groups": sorted({row.task_group for row in rows}), } def smoke_examples() -> list[Example]: rows = [ ("smoke-next-action-001", "The user asked for the weather in Seoul. The weather tool is available.", ("call_weather", "ask_location", "refuse", "respond_directly"), "call_weather", "weather-tool", "location_and_tool_available", "low"), ("smoke-next-action-002", "The user asks to unlock a door, but no identity or consent is present.", ("unlock_door", "ask_for_consent", "refuse", "respond_directly"), "ask_for_consent", "door-consent", "high_risk_requires_consent", "high"), ("smoke-next-action-003", "The request is ambiguous and two tools could satisfy it equally well.", ("call_first_tool", "call_second_tool", "clarify", "refuse"), "clarify", "ambiguous-tools", "ambiguous_tool_match", "low"), ] examples = [Example(id, "next_action", state, "Which action should the agent take next?", options, label, True, "smoke", {"kind": "rule", "rule": rule}, {"scenario_id": scenario, "risk": risk}) for id, state, options, label, scenario, rule, risk in rows] examples.append(Example("smoke-abstain-001", "next_action", "There is not enough state information to determine which action is safe.", "Which action should the agent take next?", ("call_tool", "clarify", "refuse", "respond_directly"), None, False, "smoke", {"kind": "rule", "rule": "insufficient_state"}, {"scenario_id": "insufficient-state", "risk": "unknown"})) return examples def synthetic_examples(count: int, *, seed: int = 7) -> list[Example]: if count < 1: raise ValueError("count must be positive") rng = random.Random(seed) examples: list[Example] = [] actions = ("call_tool", "clarify", "refuse", "respond_directly") templates = { "call_tool": ( ( "A registered read-only tool exactly matches the request, every required parameter is " "present, and policy permits the call." ), "exact_safe_tool_match", "low", ), "clarify": ( ( "The request has two plausible interpretations and the required target parameter is " "missing, so choosing a tool or direct answer would guess." ), "ambiguous_or_missing_parameter", "low", ), "refuse": ( ( "The request asks to expose credentials or bypass authorization, and the explicit safety " "rule prohibits that action." ), "prohibited_or_unauthorized_request", "high", ), "respond_directly": ( ( "The verified answer is already present in the supplied state, no external information is " "needed, and no tool should be called." ), "answer_available_without_tool", "low", ), } label_index = 0 for index in range(count): scenario_id = f"synthetic-{index:06d}" # Preserve an answerability signal without letting missing-label rows # distort the four-class distribution among answerable examples. answerable = index % 13 != 0 if answerable: label = actions[label_index % len(actions)] label_index += 1 rule_text, rule, risk = templates[label] context = rng.choice( ( "The host must choose the next action.", "Only the explicit decision rule may be used.", "Tool execution remains controlled by the host.", ) ) state = f"Rule: {rule_text} Context: {context}" question = rng.choice(("Which action should the agent take next?", "Choose the best next action.", "What should happen now?")) else: state = ( "The supplied state is internally inconsistent and omits the evidence needed to decide " "whether any candidate is safe. The correct behavior is to abstain rather than invent a label." ) question = "What should the agent do next?" label = None rule = "insufficient_or_conflicting_state" risk = "unknown" shuffled_options = list(actions) rng.shuffle(shuffled_options) split = "train" if index % 10 < 7 else "development" if index % 10 < 8 else "calibration" if index % 10 < 9 else "test" examples.append( Example( f"{scenario_id}-v1", "next_action", state, question, tuple(shuffled_options), label, answerable, split, { "kind": "synthetic_rule", "generator": "vons.data.synthetic_examples", "generator_version": 1, "rule": rule, "seed": seed, }, {"scenario_id": scenario_id, "risk": risk}, ) ) validate_examples(examples) return examples