from __future__ import annotations import json import math import re import tomllib from datetime import UTC, datetime from pathlib import Path from sponsor_detection.data.profile import sha256_file, write_json_atomic from sponsor_detection.model.multi_head_model import load_saved_multi_head from sponsor_detection.smart_segment_manifest import ( MODEL_MANIFEST_SCHEMA_VERSION, parse_model_manifest, verify_artifacts, ) from sponsor_detection.model.smart_segment_decoder import BILOU_LABELS def decoder_configuration(model, export: dict[str, object]) -> dict[str, object]: """Build and validate explicit category/head/decoder semantics for export.""" categories = list(model.categories) enabled_source = list(model.enabled_categories) if not categories or len(categories) != len(set(categories)): raise ValueError("model categories must be non-empty and unique") if len(enabled_source) != len(set(enabled_source)) or not set(enabled_source) <= set(categories): raise ValueError("enabled categories must be unique members of the model category order") enabled = [category for category in categories if category in set(enabled_source)] default_raw = export.get("default_threshold", 0.7) thresholds_raw = export.get("thresholds", {}) if not _valid_threshold(default_raw): raise ValueError("decoder default_threshold must be a numeric finite value in [0, 1]") if not isinstance(thresholds_raw, dict) or any(not _valid_threshold(value) for value in thresholds_raw.values()): raise ValueError("decoder category thresholds must be numeric finite values in [0, 1]") default_threshold = float(default_raw) thresholds = {str(k): float(v) for k, v in thresholds_raw.items()} if set(thresholds) - set(categories): raise ValueError("decoder thresholds contain an unknown category") merge_gap_ms = export.get("merge_gap_ms", 1500) if isinstance(merge_gap_ms, bool) or not isinstance(merge_gap_ms, int) or merge_gap_ms < 0: raise ValueError("decoder merge_gap_ms must be non-negative") preserve_window_boundaries = export.get("preserve_window_boundaries", False) if type(preserve_window_boundaries) is not bool: raise ValueError("decoder preserve_window_boundaries must be boolean") merge_gap_characters = export.get("merge_gap_characters", 24) if isinstance(merge_gap_characters, bool) or not isinstance(merge_gap_characters, int) or merge_gap_characters < 0: raise ValueError("decoder merge_gap_characters must be a non-negative integer") min_span_chars = export.get("min_span_chars", 0) if isinstance(min_span_chars, bool) or not isinstance(min_span_chars, int) or min_span_chars < 0: raise ValueError("decoder min_span_chars must be a non-negative integer") frozen = set(getattr(model, "frozen_categories", ())) if frozen - set(categories): raise ValueError("frozen categories contain an unknown category") return { "version": 2, "calibrated": False, "note": "Experimental operating point only; thresholds are not calibrated unless separately established.", "categories": categories, "labels": list(BILOU_LABELS), "default_threshold": default_threshold, "thresholds": {category: thresholds.get(category, default_threshold) for category in categories}, "merge_gap_ms": merge_gap_ms, "merge_gap_characters": merge_gap_characters, "preserve_window_boundaries": preserve_window_boundaries, "min_span_chars": min_span_chars, "enabled_categories": enabled, "trained_categories": [category for category in categories if category not in frozen], "heads": { category: { "enabled": category in enabled, "trained_in_run": category not in frozen, "frozen_in_run": category in frozen, } for category in categories }, } def validate_multi_head_interface(session, categories: list[str]) -> dict[str, object]: """Fail closed unless an ONNX graph exposes the documented rank-4 contract.""" inputs = {value.name: value for value in session.get_inputs()} if set(inputs) != {"input_ids", "attention_mask"}: raise ValueError(f"unexpected multi-head ONNX inputs: {sorted(inputs)}") for name, value in inputs.items(): if value.type != "tensor(int64)" or len(value.shape) != 2: raise ValueError(f"{name} must be int64 [batch, sequence]") outputs = {value.name: value for value in session.get_outputs()} if set(outputs) != {"segment_logits"}: raise ValueError(f"unexpected multi-head ONNX outputs: {sorted(outputs)}") output = outputs["segment_logits"] if output.type != "tensor(float)" or len(output.shape) != 4: raise ValueError("segment_logits must be rank-4 float32 [batch, sequence, category, bilou]") if output.shape[2] != len(categories) or output.shape[3] != len(BILOU_LABELS): raise ValueError("ONNX output category or BILOU axis does not match model configuration") return {"input_names": sorted(inputs), "output_name": output.name, "output_rank": len(output.shape), "category_count": output.shape[2], "bilou_count": output.shape[3]} def validate_experimental_bundle(bundle_directory: Path, *, session=None) -> dict[str, object]: """Verify hashes and rank-4 category/decoder/tokenizer contracts before evaluation.""" import hashlib bundle_directory = Path(bundle_directory) manifest_path = bundle_directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest.get("artifact_profile") != "experimental_multi_head_v2": raise ValueError("not an experimental multi-head v2 bundle") if (manifest.get("release_eligible") is not False or manifest.get("automatic_actions_enabled") is not False or manifest.get("android_compatible") is not False or manifest.get("runtime_compatibility") != "research_onnx_only_rank4_not_legacy_sponsor_rank3"): raise ValueError("experimental bundle must remain non-release, non-actionable, and Android-incompatible") parsed = parse_model_manifest(manifest) for filename in (parsed.model_file, parsed.tokenizer_file): if Path(filename).name != filename or (bundle_directory / filename).resolve().parent != bundle_directory.resolve(): raise ValueError("experimental bundle artifact path escapes its directory") verified = verify_artifacts(parsed, bundle_directory) if parsed.tokenizer_file != "tokenizer.json": raise ValueError("experimental tokenizer primary asset must be tokenizer.json") source_checkpoint = manifest.get("source_checkpoint") if (not isinstance(source_checkpoint, dict) or not isinstance(source_checkpoint.get("path"), str) or any(not isinstance(source_checkpoint.get(key), str) or not re.fullmatch(r"[0-9a-f]{64}", source_checkpoint[key]) for key in ("model_sha256", "config_sha256"))): raise ValueError("experimental bundle source-checkpoint provenance is missing or invalid") categories = [parsed.categories[index] for index in range(len(parsed.categories))] if categories != ["sponsor", "selfpromo", "interaction"]: raise ValueError("experimental bundle category order is invalid") enabled = manifest.get("enabled_categories") trained = manifest.get("trained_categories") disabled = manifest.get("disabled_categories") if not isinstance(enabled, list) or any(not isinstance(value, str) for value in enabled) or len(enabled) != len(set(enabled)) or not set(enabled) <= set(categories): raise ValueError("experimental bundle enabled category mapping is invalid") if not isinstance(trained, list) or any(not isinstance(value, str) for value in trained) or len(trained) != len(set(trained)) or not set(trained) <= set(categories): raise ValueError("experimental bundle trained category mapping is invalid") if disabled != [category for category in categories if category not in enabled]: raise ValueError("experimental bundle disabled category mapping conflicts with enabled heads") if enabled != [category for category in categories if category in set(enabled)] or trained != [category for category in categories if category in set(trained)]: raise ValueError("experimental bundle category subsets must follow the output axis order") expected_output = manifest.get("outputs", {}).get("segment_logits", {}) if expected_output.get("dtype") != "float32" or expected_output.get("shape") != ["batch", "sequence", "category", "bilou"]: raise ValueError("experimental output manifest does not declare float32 [batch, sequence, category, bilou]") if any(manifest.get("inputs", {}).get(name, {}).get("dtype") != "int64" or manifest.get("inputs", {}).get(name, {}).get("shape") != ["batch", "sequence"] for name in ("input_ids", "attention_mask")): raise ValueError("experimental inputs must declare int64 [batch, sequence]") decoder_path = bundle_directory / "decoder_config.json" expected_decoder_sha = hashlib.sha256(decoder_path.read_bytes()).hexdigest() if expected_decoder_sha != parsed.decoder_config_sha256: raise ValueError("experimental decoder configuration hash mismatch") decoder = json.loads(decoder_path.read_text(encoding="utf-8")) if decoder.get("version") != 2 or decoder.get("categories") != categories or decoder.get("labels") != list(BILOU_LABELS): raise ValueError("experimental decoder axis order is invalid") if decoder.get("enabled_categories") != enabled or decoder.get("trained_categories") != trained: raise ValueError("experimental decoder head status differs from manifest") heads = decoder.get("heads") if not isinstance(heads, dict) or set(heads) != set(categories): raise ValueError("experimental decoder head mapping is invalid") for category in categories: if not isinstance(heads[category], dict) or heads[category].get("enabled") is not (category in enabled) or heads[category].get("trained_in_run") is not (category in trained) or heads[category].get("frozen_in_run") is not (category not in trained): raise ValueError("experimental decoder head flags conflict with declared categories") thresholds = decoder.get("thresholds") default_threshold = decoder.get("default_threshold") merge_gap = decoder.get("merge_gap_ms") if not isinstance(thresholds, dict) or set(thresholds) != set(categories) or not _valid_threshold(default_threshold) or any(not _valid_threshold(v) for v in thresholds.values()): raise ValueError("experimental decoder thresholds are invalid") if isinstance(merge_gap, bool) or not isinstance(merge_gap, int) or merge_gap < 0: raise ValueError("experimental decoder merge gap is invalid") min_span_chars = decoder.get("min_span_chars", 0) if isinstance(min_span_chars, bool) or not isinstance(min_span_chars, int) or min_span_chars < 0: raise ValueError("experimental decoder min_span_chars is invalid") assets = manifest.get("tokenizer_assets") if not isinstance(assets, dict) or "tokenizer.json" not in assets: raise ValueError("experimental bundle has no pinned tokenizer asset inventory") for name, asset in assets.items(): filename = asset.get("file") if isinstance(asset, dict) else None if filename != name or Path(name).name != name or (bundle_directory / name).resolve().parent != bundle_directory.resolve(): raise ValueError("tokenizer asset path is unsafe or inconsistent") path = bundle_directory / name if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != asset.get("sha256"): raise ValueError(f"tokenizer asset hash mismatch: {name}") verified[f"tokenizer_asset:{name}"] = asset["sha256"] if assets[parsed.tokenizer_file].get("sha256") != parsed.tokenizer_sha256: raise ValueError("primary tokenizer hash differs from tokenizer asset inventory") decoder_manifest = dict(manifest.get("decoder", {})) # Legacy experimental manifests predate min_span_chars; treat a missing key # as the disabled default rather than rejecting an unchanged bundle. decoder_manifest.setdefault("min_span_chars", 0) if decoder_manifest != { "default_threshold": default_threshold, "thresholds": thresholds, "merge_gap_ms": merge_gap, "min_span_chars": min_span_chars, "enabled_categories": enabled, "category_order": categories, "label_order": list(BILOU_LABELS), }: raise ValueError("manifest decoder settings differ from decoder configuration") if session is None: import onnxruntime as ort session = ort.InferenceSession(str(bundle_directory / parsed.model_file), providers=["CPUExecutionProvider"]) interface = validate_multi_head_interface(session, categories) return {"manifest": parsed, "categories": categories, "enabled_categories": enabled, "trained_categories": trained, "disabled_categories": disabled, "decoder": decoder, "verified_artifacts": verified, "onnx_interface": interface} def _valid_threshold(value: object) -> bool: try: return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(float(value)) and 0 <= float(value) <= 1 except (TypeError, ValueError): return False def _onnx_wrapper(model): from torch import nn class LogitsOnly(nn.Module): def __init__(self): super().__init__() self.model = model def forward(self, input_ids, attention_mask): return self.model( input_ids=input_ids, attention_mask=attention_mask )["logits"] return LogitsOnly().eval() def _export_fp32(model, path: Path, *, sequence_length: int, opset_version: int) -> None: import torch wrapper = _onnx_wrapper(model) input_ids = torch.ones((1, sequence_length), dtype=torch.long) attention_mask = torch.ones((1, sequence_length), dtype=torch.long) torch.onnx.export( wrapper, (input_ids, attention_mask), str(path), input_names=["input_ids", "attention_mask"], output_names=["segment_logits"], dynamic_axes={ "input_ids": {0: "batch", 1: "sequence"}, "attention_mask": {0: "batch", 1: "sequence"}, "segment_logits": {0: "batch", 1: "sequence"}, }, opset_version=opset_version, do_constant_folding=True, dynamo=False, ) def _quantize(fp32_path: Path, int8_path: Path, op_types: list[str]) -> None: from onnxruntime.quantization import QuantType, quantize_dynamic quantize_dynamic( str(fp32_path), str(int8_path), weight_type=QuantType.QUInt8, op_types_to_quantize=list(op_types), ) def _parity(model, onnx_path: Path, tokenizer, texts: list[str], *, max_length: int, decoder: dict[str, object]) -> dict: import onnxruntime as ort import torch texts = _mixed_length_parity_texts(tokenizer, texts, max_length=max_length) encoded = tokenizer( texts, truncation=True, max_length=max_length, padding=True, return_tensors="pt", ) attention_array = encoded["attention_mask"].numpy() if len(set(int(row.sum()) for row in attention_array)) < 2 or attention_array.all(): raise ValueError("export parity texts must produce mixed lengths and actual batch padding") with torch.no_grad(): reference = model( input_ids=encoded["input_ids"], attention_mask=encoded["attention_mask"], )["logits"].numpy() session = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"]) interface = validate_multi_head_interface(session, decoder["categories"]) outputs = session.run( ["segment_logits"], { "input_ids": encoded["input_ids"].numpy(), "attention_mask": encoded["attention_mask"].numpy(), }, )[0] result = compare_export_outputs(reference, outputs, attention_array, decoder) result["parity_input_count"] = len(texts) result["onnx_interface"] = interface return result def _mixed_length_parity_texts(tokenizer, texts: list[str], *, max_length: int) -> list[str]: candidates = [str(text) for text in texts if isinstance(text, str) and text.strip()] if len(candidates) < 2: raise ValueError("at least two non-empty development parity texts are required") encoded = tokenizer(candidates, truncation=True, max_length=max_length, padding=False) grouped: dict[int, str] = {} for text, ids in zip(candidates, encoded["input_ids"]): grouped.setdefault(len(ids), text) if len(grouped) < 2: raise ValueError("development parity texts do not cover mixed tokenized lengths") lengths = sorted(grouped) chosen = [grouped[lengths[0]], grouped[lengths[-1]]] if len(lengths) > 2: chosen.insert(1, grouped[lengths[len(lengths) // 2]]) return chosen def compare_export_outputs(reference, outputs, attention_mask, decoder: dict[str, object]) -> dict[str, object]: """Compare active-token numerics and decoded spans for a padded text batch.""" import numpy as np reference = np.asarray(reference) outputs = np.asarray(outputs) attention_mask = np.asarray(attention_mask) if reference.ndim != 4 or outputs.ndim != 4 or reference.shape != outputs.shape: raise ValueError("PyTorch/ONNX logits must match as [batch, sequence, category, bilou]") if attention_mask.shape != reference.shape[:2] or not np.isin(attention_mask, [0, 1]).all(): raise ValueError("parity attention mask must match the batch/sequence axes") if np.any(np.diff(attention_mask, axis=1) > 0): raise ValueError("parity inputs must use right padding") if reference.shape[2] != len(decoder["categories"]) or reference.shape[3] != len(BILOU_LABELS): raise ValueError("PyTorch logits category or BILOU axis differs from decoder metadata") active = attention_mask.astype(bool) if not active.any(): raise ValueError("parity batch contains no attended tokens") if len(set(int(row.sum()) for row in active)) < 2 or active.all(): raise ValueError("parity batch must exercise mixed attended lengths and padding") difference = np.abs(reference - outputs)[active] argmax_equal = np.argmax(reference, axis=-1)[active] == np.argmax(outputs, axis=-1)[active] by_case = [] for index in range(reference.shape[0]): token_count = int(active[index].sum()) ref_spans = decoded_span_signature(reference[index:index + 1, :token_count], decoder) onnx_spans = decoded_span_signature(outputs[index:index + 1, :token_count], decoder) span_keys = lambda rows: [(row["category"], row["start_token"], row["end_token"], tuple(row["labels"])) for row in rows] confidence_diffs = ( [abs(float(left["confidence"]) - float(right["confidence"])) for left, right in zip(ref_spans, onnx_spans)] if span_keys(ref_spans) == span_keys(onnx_spans) else [] ) by_case.append({"attended_tokens": token_count, "decoded_span_parity": span_keys(ref_spans) == span_keys(onnx_spans), "decoded_span_confidence_max_difference": round(max(confidence_diffs), 6) if confidence_diffs else None, "reference_decoded_spans": ref_spans, "onnx_decoded_spans": onnx_spans}) return { "max_abs_logit_difference": round(float(difference.max()), 6), "mean_abs_logit_difference": round(float(difference.mean()), 6), "argmax_agreement": round(float(argmax_equal.mean()), 6), "decoded_span_parity": all(row["decoded_span_parity"] for row in by_case), "parity_case_count": len(by_case), "padded_sequence_length": int(reference.shape[1]), "cases": by_case, } def decoded_span_signature(logits, decoder: dict[str, object]) -> list[dict[str, object]]: """Apply the exact exported decoder to a single batch of model logits.""" from sponsor_detection.model.smart_segment_decoder import decode_multi_head, softmax_probabilities import numpy as np values = np.asarray(logits) if values.ndim != 4 or values.shape[0] != 1: raise ValueError("decoded parity expects [1, sequence, category, bilou] logits") if values.shape[2] != len(decoder["categories"]) or values.shape[3] != len(BILOU_LABELS): raise ValueError("decoded parity logits do not match declared axes") spans = decode_multi_head( softmax_probabilities(values[0]), categories=decoder["categories"], thresholds=decoder["thresholds"], default_threshold=decoder["default_threshold"], enabled_categories=decoder["enabled_categories"], labels=decoder["labels"], ) return [ {"category": span.category, "start_token": span.start_index, "end_token": span.end_index, "confidence": span.confidence, "labels": list(span.labels)} for span in spans ] def export_multi_head_from_config(config_path: Path) -> dict[str, object]: """Export the trained multi-head model to a versioned, hash-pinned bundle.""" import onnx with config_path.open("rb") as source: configuration = tomllib.load(source) export = configuration["export"] release = configuration["release"] checkpoint_directory = Path(export["checkpoint_directory"]) output_directory = Path(export["output_directory"]) if output_directory.exists() and any(output_directory.iterdir()): raise FileExistsError(f"refusing to overwrite non-empty export directory: {output_directory}") parity_inputs, parity_input_record = _load_development_parity_texts(export) revision = str(export["encoder_revision"]) sequence_length = int(export.get("sequence_length", 256)) opset_version = int(export.get("opset_version", 17)) if sequence_length < 2 or opset_version < 1: raise ValueError("export sequence_length and opset_version must be positive") model = load_saved_multi_head( checkpoint_directory, revision=revision, allow_legacy_architecture_fallback=bool( export.get("allow_legacy_architecture_fallback", False) ), ) categories = list(model.categories) if categories != ["sponsor", "selfpromo", "interaction"]: raise ValueError("experimental schema v2 requires category order sponsor/selfpromo/interaction") decoder = decoder_configuration(model, export) source_checkpoint = { "path": str(checkpoint_directory), "model_sha256": sha256_file(checkpoint_directory / "model.safetensors"), "config_sha256": sha256_file(checkpoint_directory / "config.json"), } if not (checkpoint_directory / "tokenizer.json").is_file(): raise ValueError("checkpoint export requires tokenizer.json to pin tokenizer identity") tokenizer = _load_tokenizer(checkpoint_directory) parity_inputs = _mixed_length_parity_texts(tokenizer, parity_inputs, max_length=sequence_length) output_directory.mkdir(parents=True, exist_ok=True) fp32_path = output_directory / "smart_segments.fp32.onnx" int8_path = output_directory / "smart_segments.int8.onnx" _export_fp32( model, fp32_path, sequence_length=sequence_length, opset_version=opset_version ) onnx.checker.check_model(str(fp32_path)) _quantize(fp32_path, int8_path, export.get("quantize_op_types", ["Gather"])) onnx.checker.check_model(str(int8_path)) tokenizer_path = output_directory / "tokenizer.json" import shutil copied_tokenizer_assets = [] for name in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "added_tokens.json", "vocab.json", "merges.txt", "spiece.model", "sentencepiece.bpe.model", "tokenizer.model"): source = checkpoint_directory / name if source.is_file(): shutil.copy2(source, output_directory / name) copied_tokenizer_assets.append(name) if "tokenizer.json" not in copied_tokenizer_assets: raise ValueError("checkpoint export requires tokenizer.json to pin tokenizer identity") fp32_parity = _parity( model, fp32_path, tokenizer, parity_inputs, max_length=sequence_length, decoder=decoder ) int8_parity = _parity( model, int8_path, tokenizer, parity_inputs, max_length=sequence_length, decoder=decoder ) if not fp32_parity["decoded_span_parity"] or not int8_parity["decoded_span_parity"]: raise ValueError("exported model decoded spans differ from PyTorch at the pinned decoder settings") decoder_path = output_directory / "decoder_config.json" decoder_path.write_text(json.dumps(decoder, indent=2, sort_keys=True) + "\n", encoding="utf-8") tokenizer_assets = { name: {"file": name, "sha256": sha256_file(output_directory / name)} for name in copied_tokenizer_assets } manifest = { "schema_version": MODEL_MANIFEST_SCHEMA_VERSION, "artifact_profile": "experimental_multi_head_v2", "model_version": str(release["model_version"]), "source_checkpoint": source_checkpoint, "encoder_revision": revision, "release_eligible": False, "automatic_actions_enabled": False, "android_compatible": False, "runtime_compatibility": "research_onnx_only_rank4_not_legacy_sponsor_rank3", "model": {"file": int8_path.name, "sha256": sha256_file(int8_path)}, "tokenizer": {"file": tokenizer_path.name, "sha256": sha256_file(tokenizer_path)}, "tokenizer_assets": tokenizer_assets, "inputs": { "input_ids": {"dtype": "int64", "shape": ["batch", "sequence"]}, "attention_mask": {"dtype": "int64", "shape": ["batch", "sequence"]}, }, "outputs": { "segment_logits": { "dtype": "float32", "shape": ["batch", "sequence", "category", "bilou"], } }, "categories": {str(index): category for index, category in enumerate(categories)}, "enabled_categories": list(decoder["enabled_categories"]), "trained_categories": list(decoder["trained_categories"]), "disabled_categories": [category for category in categories if category not in decoder["enabled_categories"]], "labels": {"0": "O", "1": "B", "2": "I", "3": "L", "4": "U"}, "normalization_version": 1, "decoder_version": 2, "decoder_config_sha256": sha256_file(decoder_path), "decoder": { "default_threshold": decoder["default_threshold"], "thresholds": decoder["thresholds"], "merge_gap_ms": decoder["merge_gap_ms"], "min_span_chars": decoder["min_span_chars"], "enabled_categories": list(decoder["enabled_categories"]), "category_order": categories, "label_order": list(BILOU_LABELS), }, "minimum_android_runtime_version": 1, } manifest_path = output_directory / "manifest.json" write_json_atomic(manifest_path, manifest) parsed = parse_model_manifest(json.loads(manifest_path.read_text(encoding="utf-8"))) verified = verify_artifacts(parsed, output_directory) verified["decoder_config"] = sha256_file(decoder_path) for name, asset in tokenizer_assets.items(): if sha256_file(output_directory / asset["file"]) != asset["sha256"]: raise ValueError(f"tokenizer asset hash mismatch: {name}") verified[f"tokenizer_asset:{name}"] = asset["sha256"] interface = int8_parity["onnx_interface"] report = { "model_version": manifest["model_version"], "source_checkpoint": source_checkpoint, "encoder_revision": revision, "artifact_profile": manifest["artifact_profile"], "release_eligible": False, "automatic_actions_enabled": False, "android_compatible": False, "runtime_compatibility": manifest["runtime_compatibility"], "categories": categories, "generated_at": datetime.now(tz=UTC).isoformat(), "artifacts": { "fp32": { "path": str(fp32_path), "bytes": fp32_path.stat().st_size, "sha256": sha256_file(fp32_path), }, "int8": { "path": str(int8_path), "bytes": int8_path.stat().st_size, "sha256": sha256_file(int8_path), }, "manifest": str(manifest_path), }, "verified_artifacts": verified, "onnx_interface": interface, "parity": {"fp32": fp32_parity, "int8": int8_parity}, "parity_input_example_count": len(parity_inputs), "parity_input": parity_input_record, "decoder": decoder, } write_json_atomic(output_directory / "export_report.json", report) return report def _load_tokenizer(checkpoint_directory: Path): from transformers import AutoTokenizer return AutoTokenizer.from_pretrained(str(checkpoint_directory)) def _load_development_parity_texts(export: dict[str, object]) -> tuple[list[str], dict[str, object]]: """Load only hash-pinned development text explicitly named by export config.""" import re if export.get("parity_input_role") != "development": raise ValueError("export parity_input_role must explicitly be development") path_value = export.get("parity_input_path") expected_hash = export.get("parity_input_sha256") column = str(export.get("parity_input_text_column", "text")) if not isinstance(path_value, str) or not path_value: raise ValueError("export requires parity_input_path") if not isinstance(expected_hash, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_hash): raise ValueError("export requires a declared lowercase parity_input_sha256") path = Path(path_value) actual_hash = sha256_file(path) if actual_hash != expected_hash: raise ValueError("development parity input hash differs from the configured hash") if path.suffix == ".parquet": import pyarrow.parquet as parquet table = parquet.read_table(path, columns=[column]).slice(0, 64) raw = table.column(column).to_pylist() elif path.suffix == ".jsonl": raw = [json.loads(line)[column] for line in path.read_text(encoding="utf-8").splitlines()[:64] if line.strip()] else: raise ValueError("development parity input must be a .parquet or .jsonl file") texts = [value for value in raw if isinstance(value, str) and value.strip()] if len(texts) < 2: raise ValueError("development parity input must contain at least two non-empty text examples") return texts, {"role": "development", "path": path_value, "sha256": actual_hash, "text_column": column, "candidate_example_count": len(texts)}