Download code/scripts/package_curated_production.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 68.9 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/scripts/package_curated_production.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/scripts/package_curated_production.py
-
curl -L -o package_curated_production.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/scripts/package_curated_production.py
68.9 kB
#!/usr/bin/env python3
"""Local production packaging, validation and promotion for the curated model.
This tool is deliberately separate from the experimental multi-head export in
``sponsor_detection.multi_head_export``. The experimental validator pins
``android_compatible=False`` and only accepts a rank-4 research graph, while the
shipped Android runtime reads a *rank-3* per-category graph (see
``reports/production_runtime_20261003/runtime_audit.md``). Rather than weaken
either contract we emit a new :data:`PRODUCTION_PROFILE` bundle and validate it
with a dedicated fail-closed validator.
Two runtime shapes are produced:
* ``rank4`` -- one graph with output ``[batch, sequence, category, bilou]``.
This is the best single Python runtime package and is never claimed to be
Android compatible.
* ``rank3`` -- one graph per category with output ``[batch, sequence, bilou]``.
The ``sponsor`` graph matches the existing Android sponsor contract exactly;
the three-category set is a versioned deployment contract for a future
Android runtime change and is explicitly *not deployed*.
Nothing here trains a model, publishes to any network, or mutates Android.
Promotion only moves an already-validated candidate inside a repository
``production/`` directory and writes an atomic ``current.json`` pointer after
checking a caller-supplied quality receipt that proves a completed full-corpus
fit.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import re
import shutil
import sys
import unicodedata
from datetime import UTC, datetime
from pathlib import Path
from typing import Callable, Mapping, Sequence
from sponsor_detection.data.profile import sha256_file, write_json_atomic
from sponsor_detection.model.smart_segment_decoder import (
BILOU_LABELS,
validate_min_span_chars,
)
from sponsor_detection.multi_head_export import (
_export_fp32,
_load_development_parity_texts,
_mixed_length_parity_texts,
_parity,
_quantize,
compare_export_outputs,
decoder_configuration,
validate_multi_head_interface,
)
PRODUCTION_SCHEMA_VERSION = 1
PRODUCTION_PROFILE = "curated_production_v1"
CANONICAL_CATEGORIES = ("sponsor", "selfpromo", "interaction")
RANK4_OUTPUT_NAME = "segment_logits"
RANK3_OUTPUT_NAME = "logits"
RANK4_OUTPUT_AXES = ("batch", "sequence", "category", "bilou")
RANK3_OUTPUT_AXES = ("batch", "sequence", "bilou")
RANK3_DEPLOYMENT_STATUS = "not_deployed_requires_android_runtime_change"
SPONSOR_DEPLOYMENT_STATUS = "packaged_not_published"
ANDROID_LABELS = ("O", "B-SPONSOR", "I-SPONSOR", "L-SPONSOR", "U-SPONSOR")
# Artifact keys are stable identifiers used across the manifest, sizes and
# parity sections so a missing record is always a hard failure.
RANK4_FP32 = "rank4_fp32"
RANK4_INT8 = "rank4_int8"
def _rank3_fp32(category: str) -> str:
return f"rank3_{category}_fp32"
def _rank3_int8(category: str) -> str:
return f"rank3_{category}_int8"
TOKENIZER_ASSET_NAMES = (
"tokenizer.json",
"tokenizer_config.json",
"special_tokens_map.json",
"added_tokens.json",
"vocab.json",
"merges.txt",
"spiece.model",
"sentencepiece.bpe.model",
"tokenizer.model",
)
_HEX64 = re.compile(r"[0-9a-f]{64}")
class ProductionBundleError(ValueError):
"""Raised when a production bundle cannot be trusted or promoted."""
def _require(condition: bool, message: str) -> None:
if not condition:
raise ProductionBundleError(message)
def _now() -> str:
return datetime.now(tz=UTC).isoformat()
def _read_json(path: Path) -> object:
try:
return json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as error:
raise ProductionBundleError(f"missing required file: {path}") from error
except ValueError as error:
raise ProductionBundleError(f"invalid JSON: {path}") from error
def _sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _is_hex64(value: object) -> bool:
return isinstance(value, str) and bool(_HEX64.fullmatch(value))
def _artifact_path(root: Path, relative: object) -> Path:
"""Resolve a bundle-relative artifact path and reject escapes."""
_require(isinstance(relative, str) and relative, "artifact path must be a non-empty string")
_require(not Path(relative).is_absolute(), f"artifact path must be bundle-relative: {relative!r}")
candidate = (root / relative).resolve()
root_resolved = root.resolve()
_require(
candidate != root_resolved and root_resolved in candidate.parents,
f"artifact path escapes the bundle: {relative!r}",
)
return candidate
def _artifact_entry(root: Path, relative: str) -> dict[str, object]:
path = _artifact_path(root, relative)
_require(path.is_file(), f"artifact is missing: {relative}")
return {
"file": relative,
"bytes": path.stat().st_size,
"sha256": sha256_file(path),
}
def _tree_sha256(directory: Path) -> str:
"""Hash a directory tree deterministically (relative path + file bytes)."""
digest = hashlib.sha256()
for path in sorted(item for item in directory.rglob("*") if item.is_file()):
digest.update(str(path.relative_to(directory)).encode("utf-8"))
digest.update(b"\0")
digest.update(path.read_bytes())
digest.update(b"\0")
return digest.hexdigest()
# ---------------------------------------------------------------------------
# Android-faithful helpers (tokenizer normalisation and decoder)
# ---------------------------------------------------------------------------
_WHITESPACE_PATTERN = re.compile(r"\s+")
_URL_PATTERN = re.compile(r"(?i)\b(?:https?://|www\.)\S+|\b\S+\.(?:com|net|org)\S*")
_NUMBER_PATTERN = re.compile(r"\b\d+(?:[.,:]\d+)*\b")
def android_normalize_cue(text: str) -> str:
"""Port of ``normalizeSponsorCue`` from Flow's SponsorTranscriptInference.kt.
Android normalises *before* tokenisation, so a Python runtime that feeds raw
captions to the Hugging Face tokenizer will differ. This helper exists to
produce the tokenizer golden vectors and to document the exact contract.
"""
lowered = text.lower().replace("’", "'")
lowered = _WHITESPACE_PATTERN.sub(" ", lowered).strip()
urls = _URL_PATTERN.sub("URL_TOKEN", lowered)
numbers = _NUMBER_PATTERN.sub("NUMBER_TOKEN", urls)
return unicodedata.normalize("NFC", numbers)
_LABEL_O = 0
_LABEL_B = 1
_LABEL_I = 2
_LABEL_L = 3
_LABEL_U = 4
_START_LABELS = (0, 1, 4)
_END_LABELS = (0, 3, 4)
_PREVIOUS_LABELS = (
(0, 3, 4),
(0, 3, 4),
(1, 2),
(1, 2),
(0, 3, 4),
)
def _log_softmax(values) -> list[float]:
import numpy as np
array = np.asarray(values, dtype="float64")
maximum = float(array.max())
exponentials = np.exp(array - maximum)
denominator = maximum + math.log(float(exponentials.sum()))
return [float(value) - denominator for value in array]
def android_viterbi_spans(
logits, offsets: Sequence[tuple[int, int]]
) -> list[dict[str, object]]:
"""Port of ``decodeSponsorBilou`` from Flow's SponsorTranscriptInference.kt.
Returns token-index spans for the tokens that carry characters. ``offsets``
is parallel to ``logits``; entries with ``end <= start`` (CLS/SEP) are
skipped exactly as Android skips empty offset ranges.
"""
total = logits.shape[0]
_require(total == len(offsets), "android decoder logits/offsets length mismatch")
token_indexes = [index for index in range(total) if offsets[index][1] > offsets[index][0]]
if not token_indexes:
return []
probabilities = [_log_softmax(logits[index]) for index in token_indexes]
paths = [[0] * len(BILOU_LABELS) for _ in probabilities]
scores = [
probabilities[0][label] if label in _START_LABELS else float("-inf")
for label in range(len(BILOU_LABELS))
]
for position in range(1, len(probabilities)):
next_scores = [0.0] * len(BILOU_LABELS)
for label in range(len(BILOU_LABELS)):
candidates = _PREVIOUS_LABELS[label]
best = candidates[0]
for candidate in candidates[1:]:
if scores[candidate] > scores[best]:
best = candidate
paths[position][label] = best
next_scores[label] = scores[best] + probabilities[position][label]
scores = next_scores
path = [0] * len(probabilities)
path[-1] = max(_END_LABELS, key=lambda label: scores[label])
for position in range(len(path) - 1, 0, -1):
path[position - 1] = paths[position][path[position]]
spans: list[dict[str, object]] = []
position = 0
while position < len(path):
label = path[position]
if label == _LABEL_O:
position += 1
continue
end_position = position
if label == _LABEL_B:
end_position += 1
while end_position < len(path) and path[end_position] == _LABEL_I:
end_position += 1
else:
_require(label == _LABEL_U, "android decoder produced an invalid tag path")
log_confidence = 0.0
for token_position in range(position, end_position + 1):
log_confidence += probabilities[token_position][path[token_position]]
confidence = math.exp(log_confidence / (end_position - position + 1))
start_token = token_indexes[position]
end_token = token_indexes[end_position]
spans.append(
{
"start_token": start_token,
"end_token": end_token,
"start_char": offsets[start_token][0],
"end_char": offsets[end_token][1],
"confidence": round(confidence, 6),
"labels": [BILOU_LABELS[path[token]] for token in range(position, end_position + 1)],
}
)
position = end_position + 1
return spans
def _android_span_keys(spans: Sequence[Mapping[str, object]]) -> list[tuple]:
return [
(int(span["start_char"]), int(span["end_char"]), tuple(span["labels"]))
for span in spans
]
# ---------------------------------------------------------------------------
# ONNX interface validators
# ---------------------------------------------------------------------------
def validate_rank3_interface(
session, *, output_name: str = RANK3_OUTPUT_NAME, bilou_count: int = 5
) -> dict[str, object]:
"""Fail closed unless a per-category rank-3 sponsor-style graph is exposed."""
inputs = {value.name: value for value in session.get_inputs()}
_require(set(inputs) == {"input_ids", "attention_mask"}, f"unexpected rank-3 ONNX inputs: {sorted(inputs)}")
for name, value in inputs.items():
_require(value.type == "tensor(int64)" and len(value.shape) == 2, f"{name} must be int64 [batch, sequence]")
outputs = {value.name: value for value in session.get_outputs()}
_require(set(outputs) == {output_name}, f"unexpected rank-3 ONNX outputs: {sorted(outputs)}")
output = outputs[output_name]
_require(output.type == "tensor(float)" and len(output.shape) == 3, "rank-3 logits must be float32 [batch, sequence, bilou]")
_require(int(output.shape[2]) == bilou_count, "rank-3 logits bilou axis does not match the BILOU label count")
return {
"input_names": sorted(inputs),
"output_name": output_name,
"output_rank": len(output.shape),
"bilou_count": int(output.shape[2]),
}
def validate_rank4_production_interface(session, categories: Sequence[str]) -> dict[str, object]:
contract = validate_multi_head_interface(session, list(categories))
contract["output_name"] = RANK4_OUTPUT_NAME
return contract
def _interface_from_record(record: object, *, rank: int, categories: Sequence[str]) -> dict[str, object]:
_require(isinstance(record, dict), "interface record must be an object")
_require(record.get("output_rank") == rank, f"recorded interface must be rank-{rank}")
_require(record.get("input_names") == ["attention_mask", "input_ids"], "recorded interface inputs are unexpected")
if rank == 4:
_require(
record.get("output_name") == RANK4_OUTPUT_NAME
and int(record.get("category_count", -1)) == len(categories)
and int(record.get("bilou_count", -1)) == len(BILOU_LABELS),
"recorded rank-4 interface does not match the category/BILOU contract",
)
else:
_require(
record.get("output_name") == RANK3_OUTPUT_NAME
and int(record.get("bilou_count", -1)) == len(BILOU_LABELS),
"recorded rank-3 interface does not match the BILOU contract",
)
return dict(record)
# ---------------------------------------------------------------------------
# Parity records
# ---------------------------------------------------------------------------
def _parity_flags(report: Mapping[str, object]) -> dict[str, object]:
cases = report.get("cases")
_require(isinstance(cases, list) and len(cases) >= 2, "parity requires at least two cases")
attended = [int(case["attended_tokens"]) for case in cases if isinstance(case, dict) and "attended_tokens" in case]
_require(len(attended) == len(cases), "every parity case must report attended_tokens")
padded = int(report.get("padded_sequence_length", 0))
_require(padded > 0, "parity must report a padded sequence length")
_require(len(set(attended)) >= 2, "parity inputs must exercise mixed attended lengths")
_require(any(length < padded for length in attended), "parity inputs must exercise real right padding")
return {
"mixed_length": len(set(attended)) >= 2,
"padding_present": any(length < padded for length in attended),
"attended_lengths": sorted(set(attended)),
}
def recorded_parity_is_sound(report: object) -> dict[str, object]:
"""Fail closed on a recorded parity report that is missing or dishonest."""
_require(isinstance(report, dict), "parity report must be an object")
_require(report.get("decoded_span_parity") is True, "parity requires decoded spans to match")
_require(report.get("mixed_length") is True, "parity requires mixed attended lengths")
_require(report.get("padding_present") is True, "parity requires real right padding")
_require(int(report.get("parity_case_count", 0)) >= 2, "parity requires at least two cases")
_require(
isinstance(report.get("max_abs_logit_difference"), (int, float))
and math.isfinite(float(report["max_abs_logit_difference"])),
"parity must report a finite max logit difference",
)
return dict(report)
def _augment_parity(report: Mapping[str, object]) -> dict[str, object]:
enriched = dict(report)
flags = _parity_flags(enriched)
enriched["mixed_length"] = flags["mixed_length"]
enriched["padding_present"] = flags["padding_present"]
enriched["attended_lengths"] = flags["attended_lengths"]
return enriched
# ---------------------------------------------------------------------------
# Packaging
# ---------------------------------------------------------------------------
def _export_category_fp32(model, category_index: int, path: Path, *, sequence_length: int, opset_version: int) -> None:
import torch
from torch import nn
class CategoryLogits(nn.Module):
def __init__(self):
super().__init__()
self.model = model
self.index = category_index
def forward(self, input_ids, attention_mask):
logits = self.model(input_ids=input_ids, attention_mask=attention_mask)["logits"]
return logits[:, :, self.index, :]
wrapper = CategoryLogits().eval()
input_ids = torch.ones((1, sequence_length), dtype=torch.long)
attention_mask = torch.ones((1, sequence_length), dtype=torch.long)
torch.onnx.export(
wrapper,
(input_ids, attention_mask),
str(path),
input_names=["input_ids", "attention_mask"],
output_names=[RANK3_OUTPUT_NAME],
dynamic_axes={
"input_ids": {0: "batch", 1: "sequence"},
"attention_mask": {0: "batch", 1: "sequence"},
RANK3_OUTPUT_NAME: {0: "batch", 1: "sequence"},
},
opset_version=opset_version,
do_constant_folding=True,
dynamo=False,
)
def _rank3_parity(
model,
category: str,
category_index: int,
onnx_path: Path,
tokenizer,
texts: list[str],
*,
max_length: int,
threshold: float,
) -> dict[str, object]:
import onnxruntime as ort
import torch
chosen = _mixed_length_parity_texts(tokenizer, texts, max_length=max_length)
encoded = tokenizer(
chosen,
truncation=True,
max_length=max_length,
padding=True,
return_offsets_mapping=True,
return_tensors="pt",
)
offsets = [[(int(start), int(end)) for start, end in row] for row in encoded["offset_mapping"].tolist()]
attention = encoded["attention_mask"].numpy()
_require(len({int(row.sum()) for row in attention}) >= 2 and not attention.all(), "rank-3 parity requires mixed padding")
with torch.no_grad():
reference = model(input_ids=encoded["input_ids"], attention_mask=encoded["attention_mask"])["logits"][
:, :, category_index, :
].numpy()
session = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
interface = validate_rank3_interface(session)
outputs = session.run(
[RANK3_OUTPUT_NAME],
{
"input_ids": encoded["input_ids"].numpy(),
"attention_mask": encoded["attention_mask"].numpy(),
},
)[0]
single_decoder = {
"categories": [category],
"labels": list(BILOU_LABELS),
"enabled_categories": [category],
"default_threshold": threshold,
"thresholds": {category: threshold},
}
result = compare_export_outputs(reference[:, :, None, :], outputs[:, :, None, :], attention, single_decoder)
# Android uses a Viterbi decoder, not the greedy research decoder, so record
# a second parity check under the exact Android tag transitions.
android_parity = True
for index in range(len(chosen)):
attended = int(attention[index].sum())
ref_spans = android_viterbi_spans(reference[index, :attended], offsets[index][:attended])
onnx_spans = android_viterbi_spans(outputs[index, :attended], offsets[index][:attended])
if _android_span_keys(ref_spans) != _android_span_keys(onnx_spans):
android_parity = False
break
result["android_viterbi_span_parity"] = android_parity
result["onnx_interface"] = interface
result["parity_input_count"] = len(chosen)
return _augment_parity(result)
def _tokenizer_goldens(checkpoint_directory: Path, output_directory: Path) -> dict[str, object]:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(str(checkpoint_directory))
inputs = (
"This episode is brought to you by Acme.",
"Visit https://example.com/deal and use code FLOW20.",
"Save 20% before 12:30 today!",
"Café naïve résumé — Unicode stays deterministic.",
)
cases = []
for text in inputs:
normalized = android_normalize_cue(text)
encoded = tokenizer(normalized, add_special_tokens=True, return_offsets_mapping=True)
cases.append(
{
"input": text,
"android_normalized": normalized,
"input_ids": [int(value) for value in encoded["input_ids"]],
"attention_mask": [int(value) for value in encoded["attention_mask"]],
"offset_mapping": [[int(start), int(end)] for start, end in encoded["offset_mapping"]],
}
)
return {"schema_version": 1, "note": "HF tokenizer applied to Android-normalized text", "cases": cases}
def _toolchain() -> dict[str, object]:
import importlib.metadata
versions: dict[str, object] = {}
for package in ("torch", "transformers", "onnx", "onnxruntime", "onnxscript", "numpy"):
try:
versions[package] = importlib.metadata.version(package)
except importlib.metadata.PackageNotFoundError:
versions[package] = None
return versions
def package_production_bundle(
*,
checkpoint_directory: Path,
output_directory: Path,
encoder_revision: str,
model_version: str,
export_mode: str = "both",
precisions: Sequence[str] = ("fp32", "int8"),
default_threshold: float = 0.7,
thresholds: Mapping[str, float] | None = None,
merge_gap_ms: int = 1500,
min_span_chars: int = 0,
merge_gap_characters: int = 24,
preserve_window_boundaries: bool = False,
sequence_length: int = 256,
opset_version: int = 17,
quantize_op_types: Sequence[str] = ("Gather",),
rank3_categories: Sequence[str] = CANONICAL_CATEGORIES,
category_split: bool = True,
parity_input_path: Path | None = None,
parity_input_sha256: str | None = None,
parity_input_text_column: str = "text",
allow_unverified_parity: bool = False,
dataset_manifest_path: Path | None = None,
training_receipt_path: Path | None = None,
load_model: Callable[..., object] | None = None,
) -> dict[str, object]:
"""Export and hash-pin a production bundle from a trained checkpoint.
The caller must supply a trained multi-head checkpoint. This function never
starts training. ``allow_unverified_parity`` produces a diagnostic bundle
(``release_eligible=False``) that promotion will reject.
"""
if load_model is None:
from sponsor_detection.model.multi_head_model import load_saved_multi_head
load_model = load_saved_multi_head
export_mode = export_mode.lower()
_require(export_mode in {"both", "rank4", "rank3"}, f"unsupported export mode: {export_mode}")
precisions = [str(value).lower() for value in precisions]
_require(precisions and set(precisions) <= {"fp32", "int8"}, "precisions must be a subset of fp32/int8")
categories = list(CANONICAL_CATEGORIES)
rank3_categories = [str(value) for value in rank3_categories]
_require(set(rank3_categories) <= set(categories), "rank3 categories must be known categories")
_require(len(rank3_categories) == len(set(rank3_categories)), "rank3 categories must be unique")
if export_mode in {"both", "rank3"}:
_require(rank3_categories, "rank3 export requires at least one category")
if "sponsor" in rank3_categories and not category_split:
raise ProductionBundleError("android sponsor compatibility requires the per-category split")
checkpoint_directory = Path(checkpoint_directory)
output_directory = Path(output_directory)
_require((checkpoint_directory / "model.safetensors").is_file(), "checkpoint model.safetensors is missing")
_require((checkpoint_directory / "config.json").is_file(), "checkpoint config.json is missing")
_require((checkpoint_directory / "tokenizer.json").is_file(), "checkpoint tokenizer.json is missing")
if output_directory.exists() and any(output_directory.iterdir()):
raise ProductionBundleError(f"refusing to overwrite non-empty production directory: {output_directory}")
model = load_model(checkpoint_directory, revision=encoder_revision, allow_legacy_architecture_fallback=False)
model_categories = [str(value) for value in getattr(model, "categories", [])]
_require(
model_categories == categories,
f"production schema requires category order {list(categories)}, got {model_categories}",
)
enabled_categories = [str(value) for value in getattr(model, "enabled_categories", categories)]
frozen_categories = [str(value) for value in getattr(model, "frozen_categories", [])]
trained_categories = [category for category in categories if category not in set(frozen_categories)]
min_span_chars = validate_min_span_chars(min_span_chars)
decoder = decoder_configuration(
model,
{
"default_threshold": default_threshold,
"thresholds": dict(thresholds or {}),
"merge_gap_ms": merge_gap_ms,
"merge_gap_characters": merge_gap_characters,
"preserve_window_boundaries": preserve_window_boundaries,
"min_span_chars": min_span_chars,
},
)
output_directory.mkdir(parents=True, exist_ok=True)
for relative in ("rank4", "rank3"):
(output_directory / relative).mkdir(exist_ok=True)
_require(parity_input_path is not None or allow_unverified_parity, "packaging requires --parity-input-path or --allow-unverified-parity")
parity_texts: list[str] = []
parity_record: dict[str, object] | None = None
if parity_input_path is not None:
parity_texts, parity_record = _load_development_parity_texts(
{
"parity_input_role": "development",
"parity_input_path": str(parity_input_path),
"parity_input_sha256": parity_input_sha256,
"parity_input_text_column": parity_input_text_column,
}
)
# Copy tokenizer assets first so the primary tokenizer hash is pinned even
# for diagnostic bundles that skip parity.
copied_assets: list[str] = []
for name in TOKENIZER_ASSET_NAMES:
source = checkpoint_directory / name
if source.is_file():
shutil.copy2(source, output_directory / name)
copied_assets.append(name)
_require("tokenizer.json" in copied_assets, "checkpoint export requires tokenizer.json")
tokenizer = _load_export_tokenizer(checkpoint_directory, len(parity_texts) > 0)
decoder_path = output_directory / "decoder_config.json"
decoder_path.write_text(json.dumps(decoder, indent=2, sort_keys=True) + "\n", encoding="utf-8")
artifacts: dict[str, dict[str, object]] = {}
parity: dict[str, object] = {}
interfaces: dict[str, object] = {"rank3": {}}
parity_verified = True
if export_mode in {"both", "rank4"}:
rank4_directory = output_directory / "rank4"
fp32_path = rank4_directory / "production_multi_head.fp32.onnx"
int8_path = rank4_directory / "production_multi_head.int8.onnx"
_export_fp32(model, fp32_path, sequence_length=sequence_length, opset_version=opset_version)
import onnx
onnx.checker.check_model(str(fp32_path))
artifacts[RANK4_FP32] = _artifact_entry(output_directory, str(fp32_path.relative_to(output_directory)))
if parity_texts:
parity[RANK4_FP32] = _augment_parity(
_parity(model, fp32_path, tokenizer, parity_texts, max_length=sequence_length, decoder=decoder)
)
else:
parity_verified = False
if "int8" in precisions:
_quantize(fp32_path, int8_path, list(quantize_op_types))
onnx.checker.check_model(str(int8_path))
artifacts[RANK4_INT8] = _artifact_entry(output_directory, str(int8_path.relative_to(output_directory)))
if parity_texts:
parity[RANK4_INT8] = _augment_parity(
_parity(model, int8_path, tokenizer, parity_texts, max_length=sequence_length, decoder=decoder)
)
else:
parity_verified = False
from onnxruntime import InferenceSession
chosen_rank4 = artifacts[RANK4_INT8]["file"] if RANK4_INT8 in artifacts else artifacts[RANK4_FP32]["file"]
session = InferenceSession(str(output_directory / chosen_rank4), providers=["CPUExecutionProvider"])
interfaces["rank4"] = validate_rank4_production_interface(session, categories)
interface_path = rank4_directory / "interface.json"
write_json_atomic(interface_path, {"schema_version": 1, "interface": interfaces["rank4"], "categories": categories})
rank3_lookup: dict[str, int] = {category: index for index, category in enumerate(categories)}
if export_mode in {"both", "rank3"}:
for category in rank3_categories:
index = rank3_lookup[category]
category_directory = output_directory / "rank3" / category
category_directory.mkdir(parents=True, exist_ok=True)
fp32_path = category_directory / f"{category}.fp32.onnx"
int8_path = category_directory / f"{category}.int8.onnx"
_export_category_fp32(model, index, fp32_path, sequence_length=sequence_length, opset_version=opset_version)
import onnx
onnx.checker.check_model(str(fp32_path))
artifacts[_rank3_fp32(category)] = _artifact_entry(output_directory, str(fp32_path.relative_to(output_directory)))
if parity_texts:
parity[_rank3_fp32(category)] = _rank3_parity(
model,
category,
index,
fp32_path,
tokenizer,
parity_texts,
max_length=sequence_length,
threshold=decoder["thresholds"][category],
)
else:
parity_verified = False
if "int8" in precisions:
_quantize(fp32_path, int8_path, list(quantize_op_types))
onnx.checker.check_model(str(int8_path))
artifacts[_rank3_int8(category)] = _artifact_entry(output_directory, str(int8_path.relative_to(output_directory)))
if parity_texts:
parity[_rank3_int8(category)] = _rank3_parity(
model,
category,
index,
int8_path,
tokenizer,
parity_texts,
max_length=sequence_length,
threshold=decoder["thresholds"][category],
)
else:
parity_verified = False
from onnxruntime import InferenceSession
chosen_path = artifacts[_rank3_int8(category)]["file"] if _rank3_int8(category) in artifacts else artifacts[_rank3_fp32(category)]["file"]
session = InferenceSession(str(output_directory / chosen_path), providers=["CPUExecutionProvider"])
interface = validate_rank3_interface(session)
interfaces["rank3"][category] = interface
write_json_atomic(
category_directory / "interface.json",
{"schema_version": 1, "interface": interface, "category": category},
)
tokenizer_assets = {
name: {"file": name, "sha256": sha256_file(output_directory / name)} for name in copied_assets
}
tokenizer_assets_path = output_directory / "tokenizer_assets.json"
write_json_atomic(
tokenizer_assets_path,
{"schema_version": 1, "primary": "tokenizer.json", "assets": tokenizer_assets},
)
tokenizer_goldens_path = output_directory / "tokenizer_goldens.json"
write_json_atomic(tokenizer_goldens_path, _tokenizer_goldens(checkpoint_directory, output_directory))
source_checkpoint = {
"path": str(checkpoint_directory),
"model_sha256": sha256_file(checkpoint_directory / "model.safetensors"),
"config_sha256": sha256_file(checkpoint_directory / "config.json"),
}
dataset_manifest_sha256 = sha256_file(dataset_manifest_path) if dataset_manifest_path else None
training_receipt_sha256 = sha256_file(training_receipt_path) if training_receipt_path else None
lineage = {
"schema_version": 1,
"created_at": _now(),
"source_checkpoint": source_checkpoint,
"encoder_revision": encoder_revision,
"architecture_source": getattr(model, "architecture_source", None),
"categories": categories,
"dataset_manifest_sha256": dataset_manifest_sha256,
"training_receipt_sha256": training_receipt_sha256,
"toolchain": _toolchain(),
"bundle_files": {name: asset["sha256"] for name, asset in tokenizer_assets.items()},
}
lineage_path = output_directory / "lineage.json"
write_json_atomic(lineage_path, lineage)
model_sizes_path = output_directory / "model_sizes.json"
write_json_atomic(
model_sizes_path,
{
"schema_version": 1,
"artifacts": {key: {"bytes": entry["bytes"], "sha256": entry["sha256"]} for key, entry in artifacts.items()},
"total_bytes": sum(int(entry["bytes"]) for entry in artifacts.values()),
},
)
rank3_contract: dict[str, object] | None = None
if export_mode in {"both", "rank3"}:
rank3_contract = {
"schema_version": 1,
"contract_version": 1,
"deployment_status": RANK3_DEPLOYMENT_STATUS,
"android_compatible": False,
"shared_encoder_replicated_across_categories": True,
"categories": rank3_categories,
"category_order": categories,
"inputs": {
"input_ids": {"dtype": "int64", "shape": ["batch", "sequence"]},
"attention_mask": {"dtype": "int64", "shape": ["batch", "sequence"]},
},
"outputs": {
"name": RANK3_OUTPUT_NAME,
"dtype": "float32",
"shape": list(RANK3_OUTPUT_AXES),
"bilou_count": len(BILOU_LABELS),
},
"per_category": {
category: {
"fp32": artifacts[_rank3_fp32(category)]["file"],
"int8": artifacts.get(_rank3_int8(category), {}).get("file"),
"output_name": RANK3_OUTPUT_NAME,
}
for category in rank3_categories
},
"decoder_reference": {
"android_viterbi_labels": list(BILOU_LABELS),
"sponsor_threshold": decoder["thresholds"].get("sponsor"),
"min_span_chars": decoder["min_span_chars"],
},
"android_requirements": {
"runtime_change_required": True,
"notes": (
"Android currently reads one rank-3 sponsor tensor. A three-category "
"runtime must select a per-category graph (or slice a rank-4 graph) and "
"apply the Android Viterbi decoder per category; no such runtime exists yet."
),
},
}
write_json_atomic(output_directory / "rank3_contract.json", rank3_contract)
sponsor_android_compatible = (
"sponsor" in rank3_categories
and "sponsor" in trained_categories
and "sponsor" in enabled_categories
and parity_verified
)
android_compatibility = {
"rank4_multi_head": {
"android_compatible": False,
"deployment_status": "python_runtime_only",
"runtime_compatibility": "python_onnx_rank4_multi_head_not_android_rank3",
"reason": "Android OnDeviceSponsorDetector asserts a rank-3 [batch, sequence, 5] output and cannot read the category axis.",
},
"rank3_sponsor": {
"android_compatible": False,
"tensor_contract_compatible": sponsor_android_compatible,
"deployment_status": SPONSOR_DEPLOYMENT_STATUS if sponsor_android_compatible else "not_packaged",
"runtime_compatibility": "android_rank3_sponsor_legacy_contract",
"reason": (
"Rank-3 sponsor graph matches the shipped Android tensor contract but has not "
"been ORT-converted, size-pinned, downloaded, or validated on-device."
),
},
"rank3_three_category": {
"android_compatible": False,
"deployment_status": RANK3_DEPLOYMENT_STATUS,
"runtime_compatibility": "versioned_rank3_per_category_contract_undeployed",
"reason": "No Android runtime consumes three per-category rank-3 graphs yet.",
},
}
manifest = {
"schema_version": PRODUCTION_SCHEMA_VERSION,
"artifact_profile": PRODUCTION_PROFILE,
"model_version": model_version,
"created_at": _now(),
"release_eligible": False,
"local_artifact_ready": bool(parity_verified),
"qualification_scope": "local_model_only_not_public_or_automatic_action_release",
"parity_verified": bool(parity_verified),
"automatic_actions_enabled": False,
"category_order": categories,
"categories": {str(index): category for index, category in enumerate(categories)},
"labels": {str(index): label for index, label in enumerate(BILOU_LABELS)},
"inputs": {
"input_ids": {"dtype": "int64", "shape": ["batch", "sequence"]},
"attention_mask": {"dtype": "int64", "shape": ["batch", "sequence"]},
},
"outputs": {
"rank4": {"name": RANK4_OUTPUT_NAME, "dtype": "float32", "shape": list(RANK4_OUTPUT_AXES)},
"rank3": {"name": RANK3_OUTPUT_NAME, "dtype": "float32", "shape": list(RANK3_OUTPUT_AXES)},
},
"enabled_categories": enabled_categories,
"trained_categories": trained_categories,
"disabled_categories": [category for category in categories if category not in set(enabled_categories)],
"decoder_config_sha256": sha256_file(decoder_path),
"decoder": {
"default_threshold": decoder["default_threshold"],
"thresholds": decoder["thresholds"],
"merge_gap_ms": decoder["merge_gap_ms"],
"merge_gap_characters": decoder.get("merge_gap_characters", 24),
"preserve_window_boundaries": decoder.get("preserve_window_boundaries", False),
"min_span_chars": decoder["min_span_chars"],
},
"lineage_sha256": sha256_file(lineage_path),
"tokenizer_assets_sha256": sha256_file(tokenizer_assets_path),
"model_sizes_sha256": sha256_file(model_sizes_path),
"rank3_contract_sha256": sha256_file(output_directory / "rank3_contract.json") if rank3_contract else None,
"tokenizer": {"file": "tokenizer.json", "sha256": sha256_file(output_directory / "tokenizer.json")},
"artifacts": artifacts,
"onnx_interfaces": interfaces,
"parity": parity,
"android_compatibility": android_compatibility,
"parity_input": parity_record,
}
write_json_atomic(output_directory / "production.json", manifest)
if sponsor_android_compatible:
_write_sponsor_android_manifest(output_directory, manifest, decoder, sequence_length, merge_gap_ms)
# Validate what we just built so a packaging mistake fails before promotion.
return validate_production_bundle(output_directory)
def _load_export_tokenizer(checkpoint_directory: Path, needed: bool):
if not needed:
return None
from transformers import AutoTokenizer
return AutoTokenizer.from_pretrained(str(checkpoint_directory))
def _write_sponsor_android_manifest(
output_directory: Path,
manifest: Mapping[str, object],
decoder: Mapping[str, object],
sequence_length: int,
merge_gap_ms: int,
) -> None:
sponsor_directory = output_directory / "rank3" / "sponsor"
lineage = json.loads((output_directory / "lineage.json").read_text(encoding="utf-8"))
package_files = {
name: _artifact_entry(output_directory, str(path.relative_to(output_directory)))
for name, path in {
"fp32": sponsor_directory / "sponsor.fp32.onnx",
"int8": sponsor_directory / "sponsor.int8.onnx",
"tokenizer": output_directory / "tokenizer.json",
"tokenizer_goldens": output_directory / "tokenizer_goldens.json",
}.items()
if path.is_file()
}
android_manifest = {
"schema_version": 1,
"name": f"curated-{manifest['model_version']}-sponsor-android-rank3",
"status": "packaged_pending_ort_conversion_and_on_device_validation",
"source": {
"checkpoint_path": lineage["source_checkpoint"]["path"],
"checkpoint_sha256": lineage["source_checkpoint"]["model_sha256"],
"base_model": "jhu-clsp/ettin-encoder-17m",
"base_model_license": "MIT",
},
"runtime": {
"format": "ONNX",
"execution_provider": "CPUExecutionProvider",
"quantization": "dynamic_int8_gather_only",
"output_name": RANK3_OUTPUT_NAME,
"output_shape": list(RANK3_OUTPUT_AXES),
},
"inference": {
"labels": list(ANDROID_LABELS),
"confidence_threshold": decoder["thresholds"].get("sponsor", decoder["default_threshold"]),
"max_length": sequence_length,
"overlap_tokens": 128,
"merge_gap_ms": merge_gap_ms,
"min_span_chars": decoder.get("min_span_chars", 0),
},
"package": {
"directory": str(sponsor_directory),
"files": package_files,
},
"android_compatibility": manifest["android_compatibility"]["rank3_sponsor"],
"notes": [
"This is the rank-3 sponsor graph only; selfpromo/interaction are not deployed.",
"ORT conversion, exact byte-size pinning and on-device tokenizer/decoder parity remain integration steps.",
],
}
write_json_atomic(sponsor_directory / "android_manifest.json", android_manifest)
# ---------------------------------------------------------------------------
# Production bundle validator (fail closed)
# ---------------------------------------------------------------------------
def validate_production_bundle(
bundle_directory: Path,
*,
sessions: Mapping[str, object] | None = None,
require_parity: bool = True,
) -> dict[str, object]:
"""Verify hashes, categories, interfaces and parity for a production bundle.
``sessions`` optionally maps a bundle-relative model path to a
session-like object (``get_inputs``/``get_outputs``). When omitted the
recorded interface contract is checked structurally; there is no bypass
that silently accepts a missing or dishonest record.
"""
bundle_directory = Path(bundle_directory)
manifest = _read_json(bundle_directory / "production.json")
_require(isinstance(manifest, dict), "production.json must be an object")
_require(manifest.get("schema_version") == PRODUCTION_SCHEMA_VERSION, "unsupported production schema version")
_require(manifest.get("artifact_profile") == PRODUCTION_PROFILE, "not a curated production bundle")
_require(manifest.get("automatic_actions_enabled") is False, "production bundles must not enable automatic actions")
_require(manifest.get("release_eligible") is False, "local packaging cannot establish public release eligibility")
_require(isinstance(manifest.get("model_version"), str) and manifest["model_version"], "model_version is required")
categories = list(CANONICAL_CATEGORIES)
_require(manifest.get("category_order") == categories, "production category order must be sponsor/selfpromo/interaction")
_require(
manifest.get("categories") == {str(index): category for index, category in enumerate(categories)},
"production categories mapping is invalid",
)
_require(
manifest.get("labels") == {str(index): label for index, label in enumerate(BILOU_LABELS)},
"production BILOU label mapping is invalid",
)
for name in ("input_ids", "attention_mask"):
entry = manifest.get("inputs", {}).get(name)
if entry != {"dtype": "int64", "shape": ["batch", "sequence"]}:
raise ProductionBundleError(f"input {name} must declare int64 [batch, sequence]")
_require(
manifest.get("outputs", {}).get("rank4", {}).get("shape") == list(RANK4_OUTPUT_AXES),
"rank-4 output shape must be [batch, sequence, category, bilou]",
)
_require(
manifest.get("outputs", {}).get("rank3", {}).get("shape") == list(RANK3_OUTPUT_AXES),
"rank-3 output shape must be [batch, sequence, bilou]",
)
enabled = manifest.get("enabled_categories")
trained = manifest.get("trained_categories")
_require(isinstance(enabled, list) and set(enabled) <= set(categories), "enabled_categories is invalid")
_require(isinstance(trained, list) and set(trained) <= set(categories), "trained_categories is invalid")
_require(
manifest.get("disabled_categories") == [category for category in categories if category not in set(enabled)],
"disabled_categories conflicts with enabled_categories",
)
android = manifest.get("android_compatibility")
_require(isinstance(android, dict), "android_compatibility section is required")
rank4_claim = android.get("rank4_multi_head", {})
_require(rank4_claim.get("android_compatible") is False, "rank-4 graph must never claim Android compatibility")
three = android.get("rank3_three_category", {})
_require(three.get("android_compatible") is False, "three-category rank-3 contract must not claim Android compatibility")
_require(
three.get("deployment_status") == RANK3_DEPLOYMENT_STATUS,
"three-category contract must stay undeployed",
)
sponsor_claim = android.get("rank3_sponsor", {})
_require(sponsor_claim.get("android_compatible") is False, "rank3 sponsor on-device compatibility has not been verified")
if sponsor_claim.get("tensor_contract_compatible"):
_require("sponsor" in trained, "sponsor graph cannot be Android compatible unless the sponsor head was trained")
_require(
"rank3" in str(sponsor_claim.get("runtime_compatibility", "")),
"rank3 sponsor compatibility must declare the rank-3 runtime contract",
)
decoder_path = bundle_directory / "decoder_config.json"
_require(decoder_path.is_file(), "decoder_config.json is required")
decoder_hash = sha256_file(decoder_path)
_require(
_is_hex64(manifest.get("decoder_config_sha256")) and manifest["decoder_config_sha256"] == decoder_hash,
"decoder configuration hash mismatch",
)
decoder = _read_json(decoder_path)
_require(isinstance(decoder, dict), "decoder_config.json must be an object")
_require(decoder.get("categories") == categories, "decoder category order is invalid")
_require(decoder.get("labels") == list(BILOU_LABELS), "decoder label order is invalid")
_require(
set(decoder.get("enabled_categories", [])) == set(enabled),
"decoder enabled categories differ from manifest",
)
_require(
set(decoder.get("trained_categories", [])) == set(trained),
"decoder trained categories differ from manifest",
)
try:
decoder_min_span = validate_min_span_chars(decoder.get("min_span_chars", 0))
except ValueError as error:
raise ProductionBundleError(f"decoder min_span_chars is invalid: {error}") from error
preserve_window_boundaries = decoder.get("preserve_window_boundaries", False)
_require(type(preserve_window_boundaries) is bool, "decoder preserve_window_boundaries must be boolean")
merge_gap_characters = decoder.get("merge_gap_characters", 24)
_require(type(merge_gap_characters) is int and merge_gap_characters >= 0, "decoder merge_gap_characters must be a non-negative integer")
manifest_decoder = manifest.get("decoder", {})
_require(isinstance(manifest_decoder, dict), "manifest decoder summary must be an object")
try:
manifest_min_span = validate_min_span_chars(manifest_decoder.get("min_span_chars", 0))
except ValueError as error:
raise ProductionBundleError(f"manifest decoder min_span_chars is invalid: {error}") from error
_require(
manifest_decoder.get("default_threshold") == decoder.get("default_threshold")
and manifest_decoder.get("thresholds") == decoder.get("thresholds")
and manifest_decoder.get("merge_gap_ms") == decoder.get("merge_gap_ms")
and manifest_decoder.get("merge_gap_characters", 24) == merge_gap_characters
and manifest_decoder.get("preserve_window_boundaries", False) == preserve_window_boundaries
and manifest_min_span == decoder_min_span,
"manifest decoder summary differs from the runtime decoder configuration",
)
lineage_path = bundle_directory / "lineage.json"
_require(lineage_path.is_file(), "lineage.json is required")
_require(manifest.get("lineage_sha256") == sha256_file(lineage_path), "lineage hash mismatch")
lineage = _read_json(lineage_path)
_require(isinstance(lineage, dict), "lineage.json must be an object")
source = lineage.get("source_checkpoint")
_require(isinstance(source, dict), "lineage source_checkpoint is required")
for key in ("model_sha256", "config_sha256"):
_require(_is_hex64(source.get(key)), f"lineage source_checkpoint.{key} must be a sha256")
_require(isinstance(lineage.get("encoder_revision"), str) and lineage["encoder_revision"], "lineage encoder_revision is required")
_require(lineage.get("dataset_manifest_sha256") is None or _is_hex64(lineage["dataset_manifest_sha256"]), "lineage dataset_manifest_sha256 is malformed")
tokenizer_assets_path = bundle_directory / "tokenizer_assets.json"
_require(tokenizer_assets_path.is_file(), "tokenizer_assets.json is required")
_require(manifest.get("tokenizer_assets_sha256") == sha256_file(tokenizer_assets_path), "tokenizer asset inventory hash mismatch")
inventory = _read_json(tokenizer_assets_path)
_require(isinstance(inventory, dict) and inventory.get("primary") == "tokenizer.json", "tokenizer asset inventory is invalid")
assets = inventory.get("assets")
_require(isinstance(assets, dict) and "tokenizer.json" in assets, "tokenizer asset inventory must pin tokenizer.json")
for name, asset in assets.items():
_require(isinstance(asset, dict) and asset.get("file") == name, "tokenizer asset entry is unsafe or inconsistent")
path = _artifact_path(bundle_directory, name)
_require(path.is_file(), f"tokenizer asset is missing: {name}")
_require(sha256_file(path) == asset.get("sha256"), f"tokenizer asset hash mismatch: {name}")
tokenizer_entry = manifest.get("tokenizer")
_require(isinstance(tokenizer_entry, dict) and tokenizer_entry.get("file") == "tokenizer.json", "manifest tokenizer entry is invalid")
_require(tokenizer_entry.get("sha256") == assets["tokenizer.json"]["sha256"], "primary tokenizer hash differs from the manifest")
artifacts = manifest.get("artifacts")
_require(isinstance(artifacts, dict) and artifacts, "artifact inventory is required")
verified_artifacts: dict[str, str] = {}
for key, entry in artifacts.items():
_require(isinstance(entry, dict), f"artifact entry {key} must be an object")
relative = entry.get("file")
path = _artifact_path(bundle_directory, relative)
_require(path.is_file(), f"artifact is missing: {relative}")
actual = sha256_file(path)
_require(actual == entry.get("sha256"), f"artifact hash mismatch: {key}")
_require(path.stat().st_size == entry.get("bytes"), f"artifact size mismatch: {key}")
verified_artifacts[key] = actual
_require(_is_hex64(manifest.get("model_sizes_sha256")) and (bundle_directory / "model_sizes.json").is_file(), "model_sizes.json is required")
_require(manifest.get("model_sizes_sha256") == sha256_file(bundle_directory / "model_sizes.json"), "model sizes hash mismatch")
interfaces = manifest.get("onnx_interfaces")
_require(isinstance(interfaces, dict), "onnx_interfaces is required")
rank4_record = interfaces.get("rank4")
if rank4_record is not None:
_require(RANK4_FP32 in artifacts, "rank4 interface declared without a rank-4 artifact")
for category, record in interfaces.get("rank3", {}).items():
_require(category in categories, f"unknown rank-3 category: {category}")
_require(_rank3_fp32(category) in artifacts, f"rank-3 interface for {category} has no artifact")
expected_keys = {key for key in artifacts}
_require(expected_keys, "no production artifacts were exported")
for key in artifacts:
entry = artifacts[key]
if rank4_record is not None and key in (RANK4_FP32, RANK4_INT8):
record = _interface_from_record(rank4_record, rank=4, categories=categories)
if sessions is not None and entry["file"] in sessions:
try:
live = validate_rank4_production_interface(sessions[entry["file"]], categories)
except ValueError as error:
raise ProductionBundleError(f"live rank-4 interface for {key} is invalid: {error}") from error
_require(live == record, "live rank-4 interface differs from the recorded contract")
else:
category = key[len("rank3_") :].rsplit("_", 1)[0] if key.startswith("rank3_") else None
_require(category in interfaces.get("rank3", {}), f"artifact {key} has no recorded rank-3 interface")
record = _interface_from_record(interfaces["rank3"][category], rank=3, categories=categories)
if sessions is not None and entry["file"] in sessions:
try:
live = validate_rank3_interface(sessions[entry["file"]])
except ValueError as error:
raise ProductionBundleError(f"live rank-3 interface for {category} is invalid: {error}") from error
_require(live == record, f"live rank-3 interface for {category} differs from the recorded contract")
rank3_contract_file = bundle_directory / "rank3_contract.json"
if interfaces.get("rank3"):
_require(rank3_contract_file.is_file(), "rank3_contract.json is required when rank-3 graphs are exported")
_require(manifest.get("rank3_contract_sha256") == sha256_file(rank3_contract_file), "rank3 contract hash mismatch")
contract = _read_json(rank3_contract_file)
_require(
isinstance(contract, dict)
and contract.get("android_compatible") is False
and contract.get("deployment_status") == RANK3_DEPLOYMENT_STATUS,
"rank3 contract must declare itself undeployed",
)
parity = manifest.get("parity")
if require_parity:
_require(manifest.get("parity_verified") is True, "production bundle parity is not verified")
_require(isinstance(parity, dict), "parity records are required")
for key in artifacts:
recorded_parity_is_sound(parity.get(key))
if manifest.get("local_artifact_ready") is not True:
raise ProductionBundleError("parity is marked unverified; local artifact is not ready")
_require((bundle_directory / "tokenizer.json").is_file(), "tokenizer.json is required")
return {
"manifest": manifest,
"categories": categories,
"enabled_categories": enabled,
"trained_categories": trained,
"decoder": decoder,
"lineage": lineage,
"verified_artifacts": verified_artifacts,
"onnx_interfaces": interfaces,
"sessions_checked": sorted(sessions) if sessions else [],
}
# ---------------------------------------------------------------------------
# Promotion
# ---------------------------------------------------------------------------
def _check_quality_receipt(receipt: object, manifest: Mapping[str, object], lineage: Mapping[str, object]) -> dict[str, object]:
_require(isinstance(receipt, dict), "quality receipt must be an object")
_require(receipt.get("schema_version") == 1, "unsupported quality receipt schema")
_require(str(receipt.get("status", "")).lower() == "pass", "quality receipt status is not pass")
checks = receipt.get("checks")
_require(isinstance(checks, dict) and checks, "quality receipt must include checks")
failed = sorted(name for name, value in checks.items() if value is not True)
_require(not failed, f"quality receipt has failing checks: {failed}")
training = receipt.get("training")
_require(isinstance(training, dict), "quality receipt must include training provenance")
_require(training.get("completed_full_corpus") is True, "quality receipt must prove a completed full-corpus fit")
_require(training.get("partial") is False, "quality receipt describes a partial fit")
_require(training.get("unverified") is False, "quality receipt is unverified")
source = lineage.get("source_checkpoint", {})
if training.get("checkpoint_model_sha256") is not None:
_require(
training["checkpoint_model_sha256"] == source.get("model_sha256"),
"quality receipt checkpoint model hash does not match the bundle lineage",
)
if training.get("checkpoint_config_sha256") is not None:
_require(
training["checkpoint_config_sha256"] == source.get("config_sha256"),
"quality receipt checkpoint config hash does not match the bundle lineage",
)
if training.get("dataset_manifest_sha256") is not None:
_require(
training["dataset_manifest_sha256"] == lineage.get("dataset_manifest_sha256"),
"quality receipt dataset manifest hash does not match the bundle lineage",
)
version = receipt.get("candidate_version")
if version is not None:
_require(version == manifest.get("model_version"), "quality receipt candidate version does not match the bundle")
_require(
isinstance(receipt.get("chosen_thresholds"), dict)
and bool(receipt["chosen_thresholds"])
and receipt["chosen_thresholds"] == manifest.get("decoder", {}).get("thresholds"),
"quality receipt thresholds differ from the runtime decoder thresholds",
)
return dict(receipt)
def _repo_root() -> Path:
return Path(__file__).resolve().parents[1]
def promote_production_bundle(
*,
candidate_directory: Path,
production_root: Path,
quality_receipt_path: Path,
version: str | None = None,
repo_root: Path | None = None,
sessions: Mapping[str, object] | None = None,
) -> dict[str, object]:
"""Atomically version and point at a validated production candidate.
The candidate must pass :func:`validate_production_bundle` and a
caller-supplied quality receipt proving a completed full-corpus fit. The
only mutation is inside ``production_root`` (``versions/``, ``backups/``
and the atomic ``current.json`` pointer); no network publication happens.
"""
repo_root = (repo_root or _repo_root()).resolve()
production_root = Path(production_root).resolve()
try:
relative = production_root.relative_to(repo_root)
except ValueError as error:
raise ProductionBundleError("production_root must live inside the repository") from error
_require(relative.parts and relative.parts[0] == "production", "production_root must be the repository production/ directory")
candidate_directory = Path(candidate_directory)
validation = validate_production_bundle(candidate_directory, sessions=sessions)
manifest = validation["manifest"]
lineage = validation["lineage"]
if str(manifest.get("model_version")) != str(version or manifest.get("model_version")):
raise ProductionBundleError("requested version differs from the candidate model_version")
receipt = _check_quality_receipt(_read_json(quality_receipt_path), manifest, lineage)
version = str(version or manifest["model_version"])
_require(bool(re.fullmatch(r"[A-Za-z0-9._-]+", version)), "version must be filename-safe")
versions_directory = production_root / "versions"
backups_directory = production_root / "backups"
version_directory = versions_directory / version
_require(not version_directory.exists(), f"production version already exists: {version}")
versions_directory.mkdir(parents=True, exist_ok=True)
backups_directory.mkdir(parents=True, exist_ok=True)
temporary = versions_directory / f".{version}.staging"
if temporary.exists():
shutil.rmtree(temporary)
shutil.copytree(candidate_directory, temporary)
os.replace(temporary, version_directory)
pointer_path = production_root / "current.json"
previous_pointer = _read_json(pointer_path) if pointer_path.is_file() else None
backup_relative: str | None = None
if previous_pointer is not None:
timestamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ")
backup_path = backups_directory / f"current-{timestamp}.json"
_require(not backup_path.exists(), "pointer backup collision; retry")
shutil.copyfile(pointer_path, backup_path)
backup_relative = str(backup_path.relative_to(production_root))
pointer = {
"schema_version": 1,
"artifact_profile": "curated_production_current_pointer_v1",
"version": version,
"bundle_directory": str(version_directory.relative_to(production_root)),
"bundle_tree_sha256": _tree_sha256(version_directory),
"model_version": manifest["model_version"],
"categories": manifest["category_order"],
"quality_receipt_sha256": sha256_file(quality_receipt_path),
"promoted_at": _now(),
"previous_version": previous_pointer.get("version") if isinstance(previous_pointer, dict) else None,
"previous_pointer_backup": backup_relative,
"automatic_actions_enabled": False,
"network_publication": "disabled",
}
write_json_atomic(pointer_path, pointer)
return {
"pointer": pointer,
"version_directory": str(version_directory),
"backup": backup_relative,
"receipt_checks": receipt["checks"],
}
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def _parse_threshold(values: Sequence[str] | None) -> dict[str, float]:
thresholds: dict[str, float] = {}
for value in values or ():
_require("=" in value, f"threshold must be category=value: {value!r}")
category, raw = value.split("=", 1)
thresholds[category.strip()] = float(raw)
return thresholds
def _parse_categories(value: str | None) -> list[str]:
if not value:
return list(CANONICAL_CATEGORIES)
return [item.strip() for item in value.split(",") if item.strip()]
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="package_curated_production",
description="Package, validate and promote a curated sponsor/selfpromo/interaction production model.",
)
commands = parser.add_subparsers(dest="command", required=True)
package = commands.add_parser("package", help="Export a trained checkpoint to a hash-pinned production bundle")
package.add_argument("--checkpoint", type=Path, required=True)
package.add_argument("--output-directory", type=Path, required=True)
package.add_argument("--encoder-revision", required=True)
package.add_argument("--model-version", required=True)
package.add_argument("--export-mode", choices=("both", "rank4", "rank3"), default="both")
package.add_argument("--precision", default="fp32,int8", help="Comma list of fp32/int8")
package.add_argument("--default-threshold", type=float, default=0.7)
package.add_argument("--threshold", action="append", default=[], help="category=value (repeatable)")
package.add_argument("--merge-gap-ms", type=int, default=1500)
package.add_argument("--merge-gap-characters", type=int, default=24)
package.add_argument("--preserve-window-boundaries", action="store_true")
package.add_argument(
"--min-span-chars",
type=int,
default=0,
help="persist a minimum decoded half-open character span length (0 disables; 12 is the reviewed candidate floor)",
)
package.add_argument("--sequence-length", type=int, default=256)
package.add_argument("--opset-version", type=int, default=17)
package.add_argument("--quantize-op-types", default="Gather", help="Comma list")
package.add_argument("--rank3-categories", default="sponsor,selfpromo,interaction")
package.add_argument("--category-split", dest="category_split", action="store_true", default=True)
package.add_argument("--no-category-split", dest="category_split", action="store_false")
package.add_argument("--parity-input-path", type=Path, default=None)
package.add_argument("--parity-input-sha256", default=None)
package.add_argument("--parity-input-text-column", default="text")
package.add_argument("--allow-unverified-parity", action="store_true")
package.add_argument("--dataset-manifest", type=Path, default=None)
package.add_argument("--training-receipt", type=Path, default=None)
validate = commands.add_parser("validate", help="Fail-closed validation of a production bundle")
validate.add_argument("--bundle-directory", type=Path, required=True)
validate.add_argument("--sessions-from-disk", action="store_true", help="Load ONNX sessions and re-check interfaces")
promote = commands.add_parser("promote", help="Atomically version and point at a validated candidate")
promote.add_argument("--candidate-directory", type=Path, required=True)
promote.add_argument("--production-root", type=Path, required=True)
promote.add_argument("--quality-receipt", type=Path, required=True)
promote.add_argument("--version", default=None)
return parser
def _load_sessions(manifest: Mapping[str, object], bundle_directory: Path) -> dict[str, object]:
from onnxruntime import InferenceSession
sessions: dict[str, object] = {}
for entry in manifest.get("artifacts", {}).values():
relative = entry["file"]
if relative.endswith(".onnx"):
sessions[relative] = InferenceSession(str(bundle_directory / relative), providers=["CPUExecutionProvider"])
return sessions
def main(arguments: Sequence[str] | None = None) -> int:
options = build_parser().parse_args(arguments)
if options.command == "package":
report = package_production_bundle(
checkpoint_directory=options.checkpoint,
output_directory=options.output_directory,
encoder_revision=options.encoder_revision,
model_version=options.model_version,
export_mode=options.export_mode,
precisions=[item.strip() for item in options.precision.split(",") if item.strip()],
default_threshold=options.default_threshold,
thresholds=_parse_threshold(options.threshold),
merge_gap_ms=options.merge_gap_ms,
merge_gap_characters=options.merge_gap_characters,
preserve_window_boundaries=options.preserve_window_boundaries,
min_span_chars=options.min_span_chars,
sequence_length=options.sequence_length,
opset_version=options.opset_version,
quantize_op_types=[item.strip() for item in options.quantize_op_types.split(",") if item.strip()],
rank3_categories=_parse_categories(options.rank3_categories),
category_split=options.category_split,
parity_input_path=options.parity_input_path,
parity_input_sha256=options.parity_input_sha256,
parity_input_text_column=options.parity_input_text_column,
allow_unverified_parity=options.allow_unverified_parity,
dataset_manifest_path=options.dataset_manifest,
training_receipt_path=options.training_receipt,
)
print(json.dumps({"verified_artifacts": report["verified_artifacts"]}, indent=2, sort_keys=True))
return 0
if options.command == "validate":
manifest = _read_json(options.bundle_directory / "production.json")
sessions = _load_sessions(manifest, options.bundle_directory) if options.sessions_from_disk else None
report = validate_production_bundle(options.bundle_directory, sessions=sessions)
print(json.dumps({"verified_artifacts": report["verified_artifacts"]}, indent=2, sort_keys=True))
return 0
if options.command == "promote":
report = promote_production_bundle(
candidate_directory=options.candidate_directory,
production_root=options.production_root,
quality_receipt_path=options.quality_receipt,
version=options.version,
)
print(json.dumps(report["pointer"], indent=2, sort_keys=True))
return 0
return 2
if __name__ == "__main__":
sys.exit(main())
#!/usr/bin/env python3
"""Local production packaging, validation and promotion for the curated model.
This tool is deliberately separate from the experimental multi-head export in
``sponsor_detection.multi_head_export``. The experimental validator pins
``android_compatible=False`` and only accepts a rank-4 research graph, while the
shipped Android runtime reads a *rank-3* per-category graph (see
``reports/production_runtime_20261003/runtime_audit.md``). Rather than weaken
either contract we emit a new :data:`PRODUCTION_PROFILE` bundle and validate it
with a dedicated fail-closed validator.
Two runtime shapes are produced:
* ``rank4`` -- one graph with output ``[batch, sequence, category, bilou]``.
This is the best single Python runtime package and is never claimed to be
Android compatible.
* ``rank3`` -- one graph per category with output ``[batch, sequence, bilou]``.
The ``sponsor`` graph matches the existing Android sponsor contract exactly;
the three-category set is a versioned deployment contract for a future
Android runtime change and is explicitly *not deployed*.
Nothing here trains a model, publishes to any network, or mutates Android.
Promotion only moves an already-validated candidate inside a repository
``production/`` directory and writes an atomic ``current.json`` pointer after
checking a caller-supplied quality receipt that proves a completed full-corpus
fit.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import re
import shutil
import sys
import unicodedata
from datetime import UTC, datetime
from pathlib import Path
from typing import Callable, Mapping, Sequence
from sponsor_detection.data.profile import sha256_file, write_json_atomic
from sponsor_detection.model.smart_segment_decoder import (
BILOU_LABELS,
validate_min_span_chars,
)
from sponsor_detection.multi_head_export import (
_export_fp32,
_load_development_parity_texts,
_mixed_length_parity_texts,
_parity,
_quantize,
compare_export_outputs,
decoder_configuration,
validate_multi_head_interface,
)
PRODUCTION_SCHEMA_VERSION = 1
PRODUCTION_PROFILE = "curated_production_v1"
CANONICAL_CATEGORIES = ("sponsor", "selfpromo", "interaction")
RANK4_OUTPUT_NAME = "segment_logits"
RANK3_OUTPUT_NAME = "logits"
RANK4_OUTPUT_AXES = ("batch", "sequence", "category", "bilou")
RANK3_OUTPUT_AXES = ("batch", "sequence", "bilou")
RANK3_DEPLOYMENT_STATUS = "not_deployed_requires_android_runtime_change"
SPONSOR_DEPLOYMENT_STATUS = "packaged_not_published"
ANDROID_LABELS = ("O", "B-SPONSOR", "I-SPONSOR", "L-SPONSOR", "U-SPONSOR")
# Artifact keys are stable identifiers used across the manifest, sizes and
# parity sections so a missing record is always a hard failure.
RANK4_FP32 = "rank4_fp32"
RANK4_INT8 = "rank4_int8"
def _rank3_fp32(category: str) -> str:
return f"rank3_{category}_fp32"
def _rank3_int8(category: str) -> str:
return f"rank3_{category}_int8"
TOKENIZER_ASSET_NAMES = (
"tokenizer.json",
"tokenizer_config.json",
"special_tokens_map.json",
"added_tokens.json",
"vocab.json",
"merges.txt",
"spiece.model",
"sentencepiece.bpe.model",
"tokenizer.model",
)
_HEX64 = re.compile(r"[0-9a-f]{64}")