CuriousDragon's picture
Publish verified Flow Smart Segments curated-category-encoder-ensemble-20261004
8fcf7c9 verified
Raw History Blame Contribute Delete
31.4 kB
from __future__ import annotations
import json
import math
import re
import tomllib
from datetime import UTC, datetime
from pathlib import Path
from sponsor_detection.data.profile import sha256_file, write_json_atomic
from sponsor_detection.model.multi_head_model import load_saved_multi_head
from sponsor_detection.smart_segment_manifest import (
MODEL_MANIFEST_SCHEMA_VERSION,
parse_model_manifest,
verify_artifacts,
)
from sponsor_detection.model.smart_segment_decoder import BILOU_LABELS
def decoder_configuration(model, export: dict[str, object]) -> dict[str, object]:
"""Build and validate explicit category/head/decoder semantics for export."""
categories = list(model.categories)
enabled_source = list(model.enabled_categories)
if not categories or len(categories) != len(set(categories)):
raise ValueError("model categories must be non-empty and unique")
if len(enabled_source) != len(set(enabled_source)) or not set(enabled_source) <= set(categories):
raise ValueError("enabled categories must be unique members of the model category order")
enabled = [category for category in categories if category in set(enabled_source)]
default_raw = export.get("default_threshold", 0.7)
thresholds_raw = export.get("thresholds", {})
if not _valid_threshold(default_raw):
raise ValueError("decoder default_threshold must be a numeric finite value in [0, 1]")
if not isinstance(thresholds_raw, dict) or any(not _valid_threshold(value) for value in thresholds_raw.values()):
raise ValueError("decoder category thresholds must be numeric finite values in [0, 1]")
default_threshold = float(default_raw)
thresholds = {str(k): float(v) for k, v in thresholds_raw.items()}
if set(thresholds) - set(categories):
raise ValueError("decoder thresholds contain an unknown category")
merge_gap_ms = export.get("merge_gap_ms", 1500)
if isinstance(merge_gap_ms, bool) or not isinstance(merge_gap_ms, int) or merge_gap_ms < 0:
raise ValueError("decoder merge_gap_ms must be non-negative")
preserve_window_boundaries = export.get("preserve_window_boundaries", False)
if type(preserve_window_boundaries) is not bool:
raise ValueError("decoder preserve_window_boundaries must be boolean")
merge_gap_characters = export.get("merge_gap_characters", 24)
if isinstance(merge_gap_characters, bool) or not isinstance(merge_gap_characters, int) or merge_gap_characters < 0:
raise ValueError("decoder merge_gap_characters must be a non-negative integer")
min_span_chars = export.get("min_span_chars", 0)
if isinstance(min_span_chars, bool) or not isinstance(min_span_chars, int) or min_span_chars < 0:
raise ValueError("decoder min_span_chars must be a non-negative integer")
frozen = set(getattr(model, "frozen_categories", ()))
if frozen - set(categories):
raise ValueError("frozen categories contain an unknown category")
return {
"version": 2,
"calibrated": False,
"note": "Experimental operating point only; thresholds are not calibrated unless separately established.",
"categories": categories,
"labels": list(BILOU_LABELS),
"default_threshold": default_threshold,
"thresholds": {category: thresholds.get(category, default_threshold) for category in categories},
"merge_gap_ms": merge_gap_ms,
"merge_gap_characters": merge_gap_characters,
"preserve_window_boundaries": preserve_window_boundaries,
"min_span_chars": min_span_chars,
"enabled_categories": enabled,
"trained_categories": [category for category in categories if category not in frozen],
"heads": {
category: {
"enabled": category in enabled,
"trained_in_run": category not in frozen,
"frozen_in_run": category in frozen,
}
for category in categories
},
}
def validate_multi_head_interface(session, categories: list[str]) -> dict[str, object]:
"""Fail closed unless an ONNX graph exposes the documented rank-4 contract."""
inputs = {value.name: value for value in session.get_inputs()}
if set(inputs) != {"input_ids", "attention_mask"}:
raise ValueError(f"unexpected multi-head ONNX inputs: {sorted(inputs)}")
for name, value in inputs.items():
if value.type != "tensor(int64)" or len(value.shape) != 2:
raise ValueError(f"{name} must be int64 [batch, sequence]")
outputs = {value.name: value for value in session.get_outputs()}
if set(outputs) != {"segment_logits"}:
raise ValueError(f"unexpected multi-head ONNX outputs: {sorted(outputs)}")
output = outputs["segment_logits"]
if output.type != "tensor(float)" or len(output.shape) != 4:
raise ValueError("segment_logits must be rank-4 float32 [batch, sequence, category, bilou]")
if output.shape[2] != len(categories) or output.shape[3] != len(BILOU_LABELS):
raise ValueError("ONNX output category or BILOU axis does not match model configuration")
return {"input_names": sorted(inputs), "output_name": output.name,
"output_rank": len(output.shape), "category_count": output.shape[2],
"bilou_count": output.shape[3]}
def validate_experimental_bundle(bundle_directory: Path, *, session=None) -> dict[str, object]:
"""Verify hashes and rank-4 category/decoder/tokenizer contracts before evaluation."""
import hashlib
bundle_directory = Path(bundle_directory)
manifest_path = bundle_directory / "manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if manifest.get("artifact_profile") != "experimental_multi_head_v2":
raise ValueError("not an experimental multi-head v2 bundle")
if (manifest.get("release_eligible") is not False
or manifest.get("automatic_actions_enabled") is not False
or manifest.get("android_compatible") is not False
or manifest.get("runtime_compatibility") != "research_onnx_only_rank4_not_legacy_sponsor_rank3"):
raise ValueError("experimental bundle must remain non-release, non-actionable, and Android-incompatible")
parsed = parse_model_manifest(manifest)
for filename in (parsed.model_file, parsed.tokenizer_file):
if Path(filename).name != filename or (bundle_directory / filename).resolve().parent != bundle_directory.resolve():
raise ValueError("experimental bundle artifact path escapes its directory")
verified = verify_artifacts(parsed, bundle_directory)
if parsed.tokenizer_file != "tokenizer.json":
raise ValueError("experimental tokenizer primary asset must be tokenizer.json")
source_checkpoint = manifest.get("source_checkpoint")
if (not isinstance(source_checkpoint, dict)
or not isinstance(source_checkpoint.get("path"), str)
or any(not isinstance(source_checkpoint.get(key), str)
or not re.fullmatch(r"[0-9a-f]{64}", source_checkpoint[key])
for key in ("model_sha256", "config_sha256"))):
raise ValueError("experimental bundle source-checkpoint provenance is missing or invalid")
categories = [parsed.categories[index] for index in range(len(parsed.categories))]
if categories != ["sponsor", "selfpromo", "interaction"]:
raise ValueError("experimental bundle category order is invalid")
enabled = manifest.get("enabled_categories")
trained = manifest.get("trained_categories")
disabled = manifest.get("disabled_categories")
if not isinstance(enabled, list) or any(not isinstance(value, str) for value in enabled) or len(enabled) != len(set(enabled)) or not set(enabled) <= set(categories):
raise ValueError("experimental bundle enabled category mapping is invalid")
if not isinstance(trained, list) or any(not isinstance(value, str) for value in trained) or len(trained) != len(set(trained)) or not set(trained) <= set(categories):
raise ValueError("experimental bundle trained category mapping is invalid")
if disabled != [category for category in categories if category not in enabled]:
raise ValueError("experimental bundle disabled category mapping conflicts with enabled heads")
if enabled != [category for category in categories if category in set(enabled)] or trained != [category for category in categories if category in set(trained)]:
raise ValueError("experimental bundle category subsets must follow the output axis order")
expected_output = manifest.get("outputs", {}).get("segment_logits", {})
if expected_output.get("dtype") != "float32" or expected_output.get("shape") != ["batch", "sequence", "category", "bilou"]:
raise ValueError("experimental output manifest does not declare float32 [batch, sequence, category, bilou]")
if any(manifest.get("inputs", {}).get(name, {}).get("dtype") != "int64" or manifest.get("inputs", {}).get(name, {}).get("shape") != ["batch", "sequence"] for name in ("input_ids", "attention_mask")):
raise ValueError("experimental inputs must declare int64 [batch, sequence]")
decoder_path = bundle_directory / "decoder_config.json"
expected_decoder_sha = hashlib.sha256(decoder_path.read_bytes()).hexdigest()
if expected_decoder_sha != parsed.decoder_config_sha256:
raise ValueError("experimental decoder configuration hash mismatch")
decoder = json.loads(decoder_path.read_text(encoding="utf-8"))
if decoder.get("version") != 2 or decoder.get("categories") != categories or decoder.get("labels") != list(BILOU_LABELS):
raise ValueError("experimental decoder axis order is invalid")
if decoder.get("enabled_categories") != enabled or decoder.get("trained_categories") != trained:
raise ValueError("experimental decoder head status differs from manifest")
heads = decoder.get("heads")
if not isinstance(heads, dict) or set(heads) != set(categories):
raise ValueError("experimental decoder head mapping is invalid")
for category in categories:
if not isinstance(heads[category], dict) or heads[category].get("enabled") is not (category in enabled) or heads[category].get("trained_in_run") is not (category in trained) or heads[category].get("frozen_in_run") is not (category not in trained):
raise ValueError("experimental decoder head flags conflict with declared categories")
thresholds = decoder.get("thresholds")
default_threshold = decoder.get("default_threshold")
merge_gap = decoder.get("merge_gap_ms")
if not isinstance(thresholds, dict) or set(thresholds) != set(categories) or not _valid_threshold(default_threshold) or any(not _valid_threshold(v) for v in thresholds.values()):
raise ValueError("experimental decoder thresholds are invalid")
if isinstance(merge_gap, bool) or not isinstance(merge_gap, int) or merge_gap < 0:
raise ValueError("experimental decoder merge gap is invalid")
min_span_chars = decoder.get("min_span_chars", 0)
if isinstance(min_span_chars, bool) or not isinstance(min_span_chars, int) or min_span_chars < 0:
raise ValueError("experimental decoder min_span_chars is invalid")
assets = manifest.get("tokenizer_assets")
if not isinstance(assets, dict) or "tokenizer.json" not in assets:
raise ValueError("experimental bundle has no pinned tokenizer asset inventory")
for name, asset in assets.items():
filename = asset.get("file") if isinstance(asset, dict) else None
if filename != name or Path(name).name != name or (bundle_directory / name).resolve().parent != bundle_directory.resolve():
raise ValueError("tokenizer asset path is unsafe or inconsistent")
path = bundle_directory / name
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != asset.get("sha256"):
raise ValueError(f"tokenizer asset hash mismatch: {name}")
verified[f"tokenizer_asset:{name}"] = asset["sha256"]
if assets[parsed.tokenizer_file].get("sha256") != parsed.tokenizer_sha256:
raise ValueError("primary tokenizer hash differs from tokenizer asset inventory")
decoder_manifest = dict(manifest.get("decoder", {}))
# Legacy experimental manifests predate min_span_chars; treat a missing key
# as the disabled default rather than rejecting an unchanged bundle.
decoder_manifest.setdefault("min_span_chars", 0)
if decoder_manifest != {
"default_threshold": default_threshold,
"thresholds": thresholds,
"merge_gap_ms": merge_gap,
"min_span_chars": min_span_chars,
"enabled_categories": enabled,
"category_order": categories,
"label_order": list(BILOU_LABELS),
}:
raise ValueError("manifest decoder settings differ from decoder configuration")
if session is None:
import onnxruntime as ort
session = ort.InferenceSession(str(bundle_directory / parsed.model_file), providers=["CPUExecutionProvider"])
interface = validate_multi_head_interface(session, categories)
return {"manifest": parsed, "categories": categories, "enabled_categories": enabled,
"trained_categories": trained, "disabled_categories": disabled,
"decoder": decoder, "verified_artifacts": verified, "onnx_interface": interface}
def _valid_threshold(value: object) -> bool:
try:
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(float(value)) and 0 <= float(value) <= 1
except (TypeError, ValueError):
return False
def _onnx_wrapper(model):
from torch import nn
class LogitsOnly(nn.Module):
def __init__(self):
super().__init__()
self.model = model
def forward(self, input_ids, attention_mask):
return self.model(
input_ids=input_ids, attention_mask=attention_mask
)["logits"]
return LogitsOnly().eval()
def _export_fp32(model, path: Path, *, sequence_length: int, opset_version: int) -> None:
import torch
wrapper = _onnx_wrapper(model)
input_ids = torch.ones((1, sequence_length), dtype=torch.long)
attention_mask = torch.ones((1, sequence_length), dtype=torch.long)
torch.onnx.export(
wrapper,
(input_ids, attention_mask),
str(path),
input_names=["input_ids", "attention_mask"],
output_names=["segment_logits"],
dynamic_axes={
"input_ids": {0: "batch", 1: "sequence"},
"attention_mask": {0: "batch", 1: "sequence"},
"segment_logits": {0: "batch", 1: "sequence"},
},
opset_version=opset_version,
do_constant_folding=True,
dynamo=False,
)
def _quantize(fp32_path: Path, int8_path: Path, op_types: list[str]) -> None:
from onnxruntime.quantization import QuantType, quantize_dynamic
quantize_dynamic(
str(fp32_path),
str(int8_path),
weight_type=QuantType.QUInt8,
op_types_to_quantize=list(op_types),
)
def _parity(model, onnx_path: Path, tokenizer, texts: list[str], *, max_length: int,
decoder: dict[str, object]) -> dict:
import onnxruntime as ort
import torch
texts = _mixed_length_parity_texts(tokenizer, texts, max_length=max_length)
encoded = tokenizer(
texts,
truncation=True,
max_length=max_length,
padding=True,
return_tensors="pt",
)
attention_array = encoded["attention_mask"].numpy()
if len(set(int(row.sum()) for row in attention_array)) < 2 or attention_array.all():
raise ValueError("export parity texts must produce mixed lengths and actual batch padding")
with torch.no_grad():
reference = model(
input_ids=encoded["input_ids"],
attention_mask=encoded["attention_mask"],
)["logits"].numpy()
session = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
interface = validate_multi_head_interface(session, decoder["categories"])
outputs = session.run(
["segment_logits"],
{
"input_ids": encoded["input_ids"].numpy(),
"attention_mask": encoded["attention_mask"].numpy(),
},
)[0]
result = compare_export_outputs(reference, outputs, attention_array, decoder)
result["parity_input_count"] = len(texts)
result["onnx_interface"] = interface
return result
def _mixed_length_parity_texts(tokenizer, texts: list[str], *, max_length: int) -> list[str]:
candidates = [str(text) for text in texts if isinstance(text, str) and text.strip()]
if len(candidates) < 2:
raise ValueError("at least two non-empty development parity texts are required")
encoded = tokenizer(candidates, truncation=True, max_length=max_length, padding=False)
grouped: dict[int, str] = {}
for text, ids in zip(candidates, encoded["input_ids"]):
grouped.setdefault(len(ids), text)
if len(grouped) < 2:
raise ValueError("development parity texts do not cover mixed tokenized lengths")
lengths = sorted(grouped)
chosen = [grouped[lengths[0]], grouped[lengths[-1]]]
if len(lengths) > 2:
chosen.insert(1, grouped[lengths[len(lengths) // 2]])
return chosen
def compare_export_outputs(reference, outputs, attention_mask, decoder: dict[str, object]) -> dict[str, object]:
"""Compare active-token numerics and decoded spans for a padded text batch."""
import numpy as np
reference = np.asarray(reference)
outputs = np.asarray(outputs)
attention_mask = np.asarray(attention_mask)
if reference.ndim != 4 or outputs.ndim != 4 or reference.shape != outputs.shape:
raise ValueError("PyTorch/ONNX logits must match as [batch, sequence, category, bilou]")
if attention_mask.shape != reference.shape[:2] or not np.isin(attention_mask, [0, 1]).all():
raise ValueError("parity attention mask must match the batch/sequence axes")
if np.any(np.diff(attention_mask, axis=1) > 0):
raise ValueError("parity inputs must use right padding")
if reference.shape[2] != len(decoder["categories"]) or reference.shape[3] != len(BILOU_LABELS):
raise ValueError("PyTorch logits category or BILOU axis differs from decoder metadata")
active = attention_mask.astype(bool)
if not active.any():
raise ValueError("parity batch contains no attended tokens")
if len(set(int(row.sum()) for row in active)) < 2 or active.all():
raise ValueError("parity batch must exercise mixed attended lengths and padding")
difference = np.abs(reference - outputs)[active]
argmax_equal = np.argmax(reference, axis=-1)[active] == np.argmax(outputs, axis=-1)[active]
by_case = []
for index in range(reference.shape[0]):
token_count = int(active[index].sum())
ref_spans = decoded_span_signature(reference[index:index + 1, :token_count], decoder)
onnx_spans = decoded_span_signature(outputs[index:index + 1, :token_count], decoder)
span_keys = lambda rows: [(row["category"], row["start_token"], row["end_token"], tuple(row["labels"])) for row in rows]
confidence_diffs = (
[abs(float(left["confidence"]) - float(right["confidence"])) for left, right in zip(ref_spans, onnx_spans)]
if span_keys(ref_spans) == span_keys(onnx_spans)
else []
)
by_case.append({"attended_tokens": token_count,
"decoded_span_parity": span_keys(ref_spans) == span_keys(onnx_spans),
"decoded_span_confidence_max_difference": round(max(confidence_diffs), 6) if confidence_diffs else None,
"reference_decoded_spans": ref_spans,
"onnx_decoded_spans": onnx_spans})
return {
"max_abs_logit_difference": round(float(difference.max()), 6),
"mean_abs_logit_difference": round(float(difference.mean()), 6),
"argmax_agreement": round(float(argmax_equal.mean()), 6),
"decoded_span_parity": all(row["decoded_span_parity"] for row in by_case),
"parity_case_count": len(by_case),
"padded_sequence_length": int(reference.shape[1]),
"cases": by_case,
}
def decoded_span_signature(logits, decoder: dict[str, object]) -> list[dict[str, object]]:
"""Apply the exact exported decoder to a single batch of model logits."""
from sponsor_detection.model.smart_segment_decoder import decode_multi_head, softmax_probabilities
import numpy as np
values = np.asarray(logits)
if values.ndim != 4 or values.shape[0] != 1:
raise ValueError("decoded parity expects [1, sequence, category, bilou] logits")
if values.shape[2] != len(decoder["categories"]) or values.shape[3] != len(BILOU_LABELS):
raise ValueError("decoded parity logits do not match declared axes")
spans = decode_multi_head(
softmax_probabilities(values[0]),
categories=decoder["categories"],
thresholds=decoder["thresholds"],
default_threshold=decoder["default_threshold"],
enabled_categories=decoder["enabled_categories"],
labels=decoder["labels"],
)
return [
{"category": span.category, "start_token": span.start_index,
"end_token": span.end_index, "confidence": span.confidence,
"labels": list(span.labels)}
for span in spans
]
def export_multi_head_from_config(config_path: Path) -> dict[str, object]:
"""Export the trained multi-head model to a versioned, hash-pinned bundle."""
import onnx
with config_path.open("rb") as source:
configuration = tomllib.load(source)
export = configuration["export"]
release = configuration["release"]
checkpoint_directory = Path(export["checkpoint_directory"])
output_directory = Path(export["output_directory"])
if output_directory.exists() and any(output_directory.iterdir()):
raise FileExistsError(f"refusing to overwrite non-empty export directory: {output_directory}")
parity_inputs, parity_input_record = _load_development_parity_texts(export)
revision = str(export["encoder_revision"])
sequence_length = int(export.get("sequence_length", 256))
opset_version = int(export.get("opset_version", 17))
if sequence_length < 2 or opset_version < 1:
raise ValueError("export sequence_length and opset_version must be positive")
model = load_saved_multi_head(
checkpoint_directory,
revision=revision,
allow_legacy_architecture_fallback=bool(
export.get("allow_legacy_architecture_fallback", False)
),
)
categories = list(model.categories)
if categories != ["sponsor", "selfpromo", "interaction"]:
raise ValueError("experimental schema v2 requires category order sponsor/selfpromo/interaction")
decoder = decoder_configuration(model, export)
source_checkpoint = {
"path": str(checkpoint_directory),
"model_sha256": sha256_file(checkpoint_directory / "model.safetensors"),
"config_sha256": sha256_file(checkpoint_directory / "config.json"),
}
if not (checkpoint_directory / "tokenizer.json").is_file():
raise ValueError("checkpoint export requires tokenizer.json to pin tokenizer identity")
tokenizer = _load_tokenizer(checkpoint_directory)
parity_inputs = _mixed_length_parity_texts(tokenizer, parity_inputs, max_length=sequence_length)
output_directory.mkdir(parents=True, exist_ok=True)
fp32_path = output_directory / "smart_segments.fp32.onnx"
int8_path = output_directory / "smart_segments.int8.onnx"
_export_fp32(
model, fp32_path, sequence_length=sequence_length, opset_version=opset_version
)
onnx.checker.check_model(str(fp32_path))
_quantize(fp32_path, int8_path, export.get("quantize_op_types", ["Gather"]))
onnx.checker.check_model(str(int8_path))
tokenizer_path = output_directory / "tokenizer.json"
import shutil
copied_tokenizer_assets = []
for name in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "added_tokens.json", "vocab.json", "merges.txt", "spiece.model", "sentencepiece.bpe.model", "tokenizer.model"):
source = checkpoint_directory / name
if source.is_file():
shutil.copy2(source, output_directory / name)
copied_tokenizer_assets.append(name)
if "tokenizer.json" not in copied_tokenizer_assets:
raise ValueError("checkpoint export requires tokenizer.json to pin tokenizer identity")
fp32_parity = _parity(
model, fp32_path, tokenizer, parity_inputs, max_length=sequence_length, decoder=decoder
)
int8_parity = _parity(
model, int8_path, tokenizer, parity_inputs, max_length=sequence_length, decoder=decoder
)
if not fp32_parity["decoded_span_parity"] or not int8_parity["decoded_span_parity"]:
raise ValueError("exported model decoded spans differ from PyTorch at the pinned decoder settings")
decoder_path = output_directory / "decoder_config.json"
decoder_path.write_text(json.dumps(decoder, indent=2, sort_keys=True) + "\n", encoding="utf-8")
tokenizer_assets = {
name: {"file": name, "sha256": sha256_file(output_directory / name)}
for name in copied_tokenizer_assets
}
manifest = {
"schema_version": MODEL_MANIFEST_SCHEMA_VERSION,
"artifact_profile": "experimental_multi_head_v2",
"model_version": str(release["model_version"]),
"source_checkpoint": source_checkpoint,
"encoder_revision": revision,
"release_eligible": False,
"automatic_actions_enabled": False,
"android_compatible": False,
"runtime_compatibility": "research_onnx_only_rank4_not_legacy_sponsor_rank3",
"model": {"file": int8_path.name, "sha256": sha256_file(int8_path)},
"tokenizer": {"file": tokenizer_path.name, "sha256": sha256_file(tokenizer_path)},
"tokenizer_assets": tokenizer_assets,
"inputs": {
"input_ids": {"dtype": "int64", "shape": ["batch", "sequence"]},
"attention_mask": {"dtype": "int64", "shape": ["batch", "sequence"]},
},
"outputs": {
"segment_logits": {
"dtype": "float32",
"shape": ["batch", "sequence", "category", "bilou"],
}
},
"categories": {str(index): category for index, category in enumerate(categories)},
"enabled_categories": list(decoder["enabled_categories"]),
"trained_categories": list(decoder["trained_categories"]),
"disabled_categories": [category for category in categories if category not in decoder["enabled_categories"]],
"labels": {"0": "O", "1": "B", "2": "I", "3": "L", "4": "U"},
"normalization_version": 1,
"decoder_version": 2,
"decoder_config_sha256": sha256_file(decoder_path),
"decoder": {
"default_threshold": decoder["default_threshold"],
"thresholds": decoder["thresholds"],
"merge_gap_ms": decoder["merge_gap_ms"],
"min_span_chars": decoder["min_span_chars"],
"enabled_categories": list(decoder["enabled_categories"]),
"category_order": categories,
"label_order": list(BILOU_LABELS),
},
"minimum_android_runtime_version": 1,
}
manifest_path = output_directory / "manifest.json"
write_json_atomic(manifest_path, manifest)
parsed = parse_model_manifest(json.loads(manifest_path.read_text(encoding="utf-8")))
verified = verify_artifacts(parsed, output_directory)
verified["decoder_config"] = sha256_file(decoder_path)
for name, asset in tokenizer_assets.items():
if sha256_file(output_directory / asset["file"]) != asset["sha256"]:
raise ValueError(f"tokenizer asset hash mismatch: {name}")
verified[f"tokenizer_asset:{name}"] = asset["sha256"]
interface = int8_parity["onnx_interface"]
report = {
"model_version": manifest["model_version"],
"source_checkpoint": source_checkpoint,
"encoder_revision": revision,
"artifact_profile": manifest["artifact_profile"],
"release_eligible": False,
"automatic_actions_enabled": False,
"android_compatible": False,
"runtime_compatibility": manifest["runtime_compatibility"],
"categories": categories,
"generated_at": datetime.now(tz=UTC).isoformat(),
"artifacts": {
"fp32": {
"path": str(fp32_path),
"bytes": fp32_path.stat().st_size,
"sha256": sha256_file(fp32_path),
},
"int8": {
"path": str(int8_path),
"bytes": int8_path.stat().st_size,
"sha256": sha256_file(int8_path),
},
"manifest": str(manifest_path),
},
"verified_artifacts": verified,
"onnx_interface": interface,
"parity": {"fp32": fp32_parity, "int8": int8_parity},
"parity_input_example_count": len(parity_inputs),
"parity_input": parity_input_record,
"decoder": decoder,
}
write_json_atomic(output_directory / "export_report.json", report)
return report
def _load_tokenizer(checkpoint_directory: Path):
from transformers import AutoTokenizer
return AutoTokenizer.from_pretrained(str(checkpoint_directory))
def _load_development_parity_texts(export: dict[str, object]) -> tuple[list[str], dict[str, object]]:
"""Load only hash-pinned development text explicitly named by export config."""
import re
if export.get("parity_input_role") != "development":
raise ValueError("export parity_input_role must explicitly be development")
path_value = export.get("parity_input_path")
expected_hash = export.get("parity_input_sha256")
column = str(export.get("parity_input_text_column", "text"))
if not isinstance(path_value, str) or not path_value:
raise ValueError("export requires parity_input_path")
if not isinstance(expected_hash, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_hash):
raise ValueError("export requires a declared lowercase parity_input_sha256")
path = Path(path_value)
actual_hash = sha256_file(path)
if actual_hash != expected_hash:
raise ValueError("development parity input hash differs from the configured hash")
if path.suffix == ".parquet":
import pyarrow.parquet as parquet
table = parquet.read_table(path, columns=[column]).slice(0, 64)
raw = table.column(column).to_pylist()
elif path.suffix == ".jsonl":
raw = [json.loads(line)[column] for line in path.read_text(encoding="utf-8").splitlines()[:64] if line.strip()]
else:
raise ValueError("development parity input must be a .parquet or .jsonl file")
texts = [value for value in raw if isinstance(value, str) and value.strip()]
if len(texts) < 2:
raise ValueError("development parity input must contain at least two non-empty text examples")
return texts, {"role": "development", "path": path_value, "sha256": actual_hash,
"text_column": column, "candidate_example_count": len(texts)}