Download code/src/sponsor_detection/multi_head_export.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 31.4 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/multi_head_export.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/src/sponsor_detection/multi_head_export.py
-
curl -L -o multi_head_export.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/multi_head_export.py
31.4 kB
| from __future__ import annotations | |
| import json | |
| import math | |
| import re | |
| import tomllib | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from sponsor_detection.data.profile import sha256_file, write_json_atomic | |
| from sponsor_detection.model.multi_head_model import load_saved_multi_head | |
| from sponsor_detection.smart_segment_manifest import ( | |
| MODEL_MANIFEST_SCHEMA_VERSION, | |
| parse_model_manifest, | |
| verify_artifacts, | |
| ) | |
| from sponsor_detection.model.smart_segment_decoder import BILOU_LABELS | |
| def decoder_configuration(model, export: dict[str, object]) -> dict[str, object]: | |
| """Build and validate explicit category/head/decoder semantics for export.""" | |
| categories = list(model.categories) | |
| enabled_source = list(model.enabled_categories) | |
| if not categories or len(categories) != len(set(categories)): | |
| raise ValueError("model categories must be non-empty and unique") | |
| if len(enabled_source) != len(set(enabled_source)) or not set(enabled_source) <= set(categories): | |
| raise ValueError("enabled categories must be unique members of the model category order") | |
| enabled = [category for category in categories if category in set(enabled_source)] | |
| default_raw = export.get("default_threshold", 0.7) | |
| thresholds_raw = export.get("thresholds", {}) | |
| if not _valid_threshold(default_raw): | |
| raise ValueError("decoder default_threshold must be a numeric finite value in [0, 1]") | |
| if not isinstance(thresholds_raw, dict) or any(not _valid_threshold(value) for value in thresholds_raw.values()): | |
| raise ValueError("decoder category thresholds must be numeric finite values in [0, 1]") | |
| default_threshold = float(default_raw) | |
| thresholds = {str(k): float(v) for k, v in thresholds_raw.items()} | |
| if set(thresholds) - set(categories): | |
| raise ValueError("decoder thresholds contain an unknown category") | |
| merge_gap_ms = export.get("merge_gap_ms", 1500) | |
| if isinstance(merge_gap_ms, bool) or not isinstance(merge_gap_ms, int) or merge_gap_ms < 0: | |
| raise ValueError("decoder merge_gap_ms must be non-negative") | |
| preserve_window_boundaries = export.get("preserve_window_boundaries", False) | |
| if type(preserve_window_boundaries) is not bool: | |
| raise ValueError("decoder preserve_window_boundaries must be boolean") | |
| merge_gap_characters = export.get("merge_gap_characters", 24) | |
| if isinstance(merge_gap_characters, bool) or not isinstance(merge_gap_characters, int) or merge_gap_characters < 0: | |
| raise ValueError("decoder merge_gap_characters must be a non-negative integer") | |
| min_span_chars = export.get("min_span_chars", 0) | |
| if isinstance(min_span_chars, bool) or not isinstance(min_span_chars, int) or min_span_chars < 0: | |
| raise ValueError("decoder min_span_chars must be a non-negative integer") | |
| frozen = set(getattr(model, "frozen_categories", ())) | |
| if frozen - set(categories): | |
| raise ValueError("frozen categories contain an unknown category") | |
| return { | |
| "version": 2, | |
| "calibrated": False, | |
| "note": "Experimental operating point only; thresholds are not calibrated unless separately established.", | |
| "categories": categories, | |
| "labels": list(BILOU_LABELS), | |
| "default_threshold": default_threshold, | |
| "thresholds": {category: thresholds.get(category, default_threshold) for category in categories}, | |
| "merge_gap_ms": merge_gap_ms, | |
| "merge_gap_characters": merge_gap_characters, | |
| "preserve_window_boundaries": preserve_window_boundaries, | |
| "min_span_chars": min_span_chars, | |
| "enabled_categories": enabled, | |
| "trained_categories": [category for category in categories if category not in frozen], | |
| "heads": { | |
| category: { | |
| "enabled": category in enabled, | |
| "trained_in_run": category not in frozen, | |
| "frozen_in_run": category in frozen, | |
| } | |
| for category in categories | |
| }, | |
| } | |
| def validate_multi_head_interface(session, categories: list[str]) -> dict[str, object]: | |
| """Fail closed unless an ONNX graph exposes the documented rank-4 contract.""" | |
| inputs = {value.name: value for value in session.get_inputs()} | |
| if set(inputs) != {"input_ids", "attention_mask"}: | |
| raise ValueError(f"unexpected multi-head ONNX inputs: {sorted(inputs)}") | |
| for name, value in inputs.items(): | |
| if value.type != "tensor(int64)" or len(value.shape) != 2: | |
| raise ValueError(f"{name} must be int64 [batch, sequence]") | |
| outputs = {value.name: value for value in session.get_outputs()} | |
| if set(outputs) != {"segment_logits"}: | |
| raise ValueError(f"unexpected multi-head ONNX outputs: {sorted(outputs)}") | |
| output = outputs["segment_logits"] | |
| if output.type != "tensor(float)" or len(output.shape) != 4: | |
| raise ValueError("segment_logits must be rank-4 float32 [batch, sequence, category, bilou]") | |
| if output.shape[2] != len(categories) or output.shape[3] != len(BILOU_LABELS): | |
| raise ValueError("ONNX output category or BILOU axis does not match model configuration") | |
| return {"input_names": sorted(inputs), "output_name": output.name, | |
| "output_rank": len(output.shape), "category_count": output.shape[2], | |
| "bilou_count": output.shape[3]} | |
| def validate_experimental_bundle(bundle_directory: Path, *, session=None) -> dict[str, object]: | |
| """Verify hashes and rank-4 category/decoder/tokenizer contracts before evaluation.""" | |
| import hashlib | |
| bundle_directory = Path(bundle_directory) | |
| manifest_path = bundle_directory / "manifest.json" | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| if manifest.get("artifact_profile") != "experimental_multi_head_v2": | |
| raise ValueError("not an experimental multi-head v2 bundle") | |
| if (manifest.get("release_eligible") is not False | |
| or manifest.get("automatic_actions_enabled") is not False | |
| or manifest.get("android_compatible") is not False | |
| or manifest.get("runtime_compatibility") != "research_onnx_only_rank4_not_legacy_sponsor_rank3"): | |
| raise ValueError("experimental bundle must remain non-release, non-actionable, and Android-incompatible") | |
| parsed = parse_model_manifest(manifest) | |
| for filename in (parsed.model_file, parsed.tokenizer_file): | |
| if Path(filename).name != filename or (bundle_directory / filename).resolve().parent != bundle_directory.resolve(): | |
| raise ValueError("experimental bundle artifact path escapes its directory") | |
| verified = verify_artifacts(parsed, bundle_directory) | |
| if parsed.tokenizer_file != "tokenizer.json": | |
| raise ValueError("experimental tokenizer primary asset must be tokenizer.json") | |
| source_checkpoint = manifest.get("source_checkpoint") | |
| if (not isinstance(source_checkpoint, dict) | |
| or not isinstance(source_checkpoint.get("path"), str) | |
| or any(not isinstance(source_checkpoint.get(key), str) | |
| or not re.fullmatch(r"[0-9a-f]{64}", source_checkpoint[key]) | |
| for key in ("model_sha256", "config_sha256"))): | |
| raise ValueError("experimental bundle source-checkpoint provenance is missing or invalid") | |
| categories = [parsed.categories[index] for index in range(len(parsed.categories))] | |
| if categories != ["sponsor", "selfpromo", "interaction"]: | |
| raise ValueError("experimental bundle category order is invalid") | |
| enabled = manifest.get("enabled_categories") | |
| trained = manifest.get("trained_categories") | |
| disabled = manifest.get("disabled_categories") | |
| if not isinstance(enabled, list) or any(not isinstance(value, str) for value in enabled) or len(enabled) != len(set(enabled)) or not set(enabled) <= set(categories): | |
| raise ValueError("experimental bundle enabled category mapping is invalid") | |
| if not isinstance(trained, list) or any(not isinstance(value, str) for value in trained) or len(trained) != len(set(trained)) or not set(trained) <= set(categories): | |
| raise ValueError("experimental bundle trained category mapping is invalid") | |
| if disabled != [category for category in categories if category not in enabled]: | |
| raise ValueError("experimental bundle disabled category mapping conflicts with enabled heads") | |
| if enabled != [category for category in categories if category in set(enabled)] or trained != [category for category in categories if category in set(trained)]: | |
| raise ValueError("experimental bundle category subsets must follow the output axis order") | |
| expected_output = manifest.get("outputs", {}).get("segment_logits", {}) | |
| if expected_output.get("dtype") != "float32" or expected_output.get("shape") != ["batch", "sequence", "category", "bilou"]: | |
| raise ValueError("experimental output manifest does not declare float32 [batch, sequence, category, bilou]") | |
| if any(manifest.get("inputs", {}).get(name, {}).get("dtype") != "int64" or manifest.get("inputs", {}).get(name, {}).get("shape") != ["batch", "sequence"] for name in ("input_ids", "attention_mask")): | |
| raise ValueError("experimental inputs must declare int64 [batch, sequence]") | |
| decoder_path = bundle_directory / "decoder_config.json" | |
| expected_decoder_sha = hashlib.sha256(decoder_path.read_bytes()).hexdigest() | |
| if expected_decoder_sha != parsed.decoder_config_sha256: | |
| raise ValueError("experimental decoder configuration hash mismatch") | |
| decoder = json.loads(decoder_path.read_text(encoding="utf-8")) | |
| if decoder.get("version") != 2 or decoder.get("categories") != categories or decoder.get("labels") != list(BILOU_LABELS): | |
| raise ValueError("experimental decoder axis order is invalid") | |
| if decoder.get("enabled_categories") != enabled or decoder.get("trained_categories") != trained: | |
| raise ValueError("experimental decoder head status differs from manifest") | |
| heads = decoder.get("heads") | |
| if not isinstance(heads, dict) or set(heads) != set(categories): | |
| raise ValueError("experimental decoder head mapping is invalid") | |
| for category in categories: | |
| if not isinstance(heads[category], dict) or heads[category].get("enabled") is not (category in enabled) or heads[category].get("trained_in_run") is not (category in trained) or heads[category].get("frozen_in_run") is not (category not in trained): | |
| raise ValueError("experimental decoder head flags conflict with declared categories") | |
| thresholds = decoder.get("thresholds") | |
| default_threshold = decoder.get("default_threshold") | |
| merge_gap = decoder.get("merge_gap_ms") | |
| if not isinstance(thresholds, dict) or set(thresholds) != set(categories) or not _valid_threshold(default_threshold) or any(not _valid_threshold(v) for v in thresholds.values()): | |
| raise ValueError("experimental decoder thresholds are invalid") | |
| if isinstance(merge_gap, bool) or not isinstance(merge_gap, int) or merge_gap < 0: | |
| raise ValueError("experimental decoder merge gap is invalid") | |
| min_span_chars = decoder.get("min_span_chars", 0) | |
| if isinstance(min_span_chars, bool) or not isinstance(min_span_chars, int) or min_span_chars < 0: | |
| raise ValueError("experimental decoder min_span_chars is invalid") | |
| assets = manifest.get("tokenizer_assets") | |
| if not isinstance(assets, dict) or "tokenizer.json" not in assets: | |
| raise ValueError("experimental bundle has no pinned tokenizer asset inventory") | |
| for name, asset in assets.items(): | |
| filename = asset.get("file") if isinstance(asset, dict) else None | |
| if filename != name or Path(name).name != name or (bundle_directory / name).resolve().parent != bundle_directory.resolve(): | |
| raise ValueError("tokenizer asset path is unsafe or inconsistent") | |
| path = bundle_directory / name | |
| if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != asset.get("sha256"): | |
| raise ValueError(f"tokenizer asset hash mismatch: {name}") | |
| verified[f"tokenizer_asset:{name}"] = asset["sha256"] | |
| if assets[parsed.tokenizer_file].get("sha256") != parsed.tokenizer_sha256: | |
| raise ValueError("primary tokenizer hash differs from tokenizer asset inventory") | |
| decoder_manifest = dict(manifest.get("decoder", {})) | |
| # Legacy experimental manifests predate min_span_chars; treat a missing key | |
| # as the disabled default rather than rejecting an unchanged bundle. | |
| decoder_manifest.setdefault("min_span_chars", 0) | |
| if decoder_manifest != { | |
| "default_threshold": default_threshold, | |
| "thresholds": thresholds, | |
| "merge_gap_ms": merge_gap, | |
| "min_span_chars": min_span_chars, | |
| "enabled_categories": enabled, | |
| "category_order": categories, | |
| "label_order": list(BILOU_LABELS), | |
| }: | |
| raise ValueError("manifest decoder settings differ from decoder configuration") | |
| if session is None: | |
| import onnxruntime as ort | |
| session = ort.InferenceSession(str(bundle_directory / parsed.model_file), providers=["CPUExecutionProvider"]) | |
| interface = validate_multi_head_interface(session, categories) | |
| return {"manifest": parsed, "categories": categories, "enabled_categories": enabled, | |
| "trained_categories": trained, "disabled_categories": disabled, | |
| "decoder": decoder, "verified_artifacts": verified, "onnx_interface": interface} | |
| def _valid_threshold(value: object) -> bool: | |
| try: | |
| return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(float(value)) and 0 <= float(value) <= 1 | |
| except (TypeError, ValueError): | |
| return False | |
| def _onnx_wrapper(model): | |
| from torch import nn | |
| class LogitsOnly(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.model = model | |
| def forward(self, input_ids, attention_mask): | |
| return self.model( | |
| input_ids=input_ids, attention_mask=attention_mask | |
| )["logits"] | |
| return LogitsOnly().eval() | |
| def _export_fp32(model, path: Path, *, sequence_length: int, opset_version: int) -> None: | |
| import torch | |
| wrapper = _onnx_wrapper(model) | |
| input_ids = torch.ones((1, sequence_length), dtype=torch.long) | |
| attention_mask = torch.ones((1, sequence_length), dtype=torch.long) | |
| torch.onnx.export( | |
| wrapper, | |
| (input_ids, attention_mask), | |
| str(path), | |
| input_names=["input_ids", "attention_mask"], | |
| output_names=["segment_logits"], | |
| dynamic_axes={ | |
| "input_ids": {0: "batch", 1: "sequence"}, | |
| "attention_mask": {0: "batch", 1: "sequence"}, | |
| "segment_logits": {0: "batch", 1: "sequence"}, | |
| }, | |
| opset_version=opset_version, | |
| do_constant_folding=True, | |
| dynamo=False, | |
| ) | |
| def _quantize(fp32_path: Path, int8_path: Path, op_types: list[str]) -> None: | |
| from onnxruntime.quantization import QuantType, quantize_dynamic | |
| quantize_dynamic( | |
| str(fp32_path), | |
| str(int8_path), | |
| weight_type=QuantType.QUInt8, | |
| op_types_to_quantize=list(op_types), | |
| ) | |
| def _parity(model, onnx_path: Path, tokenizer, texts: list[str], *, max_length: int, | |
| decoder: dict[str, object]) -> dict: | |
| import onnxruntime as ort | |
| import torch | |
| texts = _mixed_length_parity_texts(tokenizer, texts, max_length=max_length) | |
| encoded = tokenizer( | |
| texts, | |
| truncation=True, | |
| max_length=max_length, | |
| padding=True, | |
| return_tensors="pt", | |
| ) | |
| attention_array = encoded["attention_mask"].numpy() | |
| if len(set(int(row.sum()) for row in attention_array)) < 2 or attention_array.all(): | |
| raise ValueError("export parity texts must produce mixed lengths and actual batch padding") | |
| with torch.no_grad(): | |
| reference = model( | |
| input_ids=encoded["input_ids"], | |
| attention_mask=encoded["attention_mask"], | |
| )["logits"].numpy() | |
| session = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"]) | |
| interface = validate_multi_head_interface(session, decoder["categories"]) | |
| outputs = session.run( | |
| ["segment_logits"], | |
| { | |
| "input_ids": encoded["input_ids"].numpy(), | |
| "attention_mask": encoded["attention_mask"].numpy(), | |
| }, | |
| )[0] | |
| result = compare_export_outputs(reference, outputs, attention_array, decoder) | |
| result["parity_input_count"] = len(texts) | |
| result["onnx_interface"] = interface | |
| return result | |
| def _mixed_length_parity_texts(tokenizer, texts: list[str], *, max_length: int) -> list[str]: | |
| candidates = [str(text) for text in texts if isinstance(text, str) and text.strip()] | |
| if len(candidates) < 2: | |
| raise ValueError("at least two non-empty development parity texts are required") | |
| encoded = tokenizer(candidates, truncation=True, max_length=max_length, padding=False) | |
| grouped: dict[int, str] = {} | |
| for text, ids in zip(candidates, encoded["input_ids"]): | |
| grouped.setdefault(len(ids), text) | |
| if len(grouped) < 2: | |
| raise ValueError("development parity texts do not cover mixed tokenized lengths") | |
| lengths = sorted(grouped) | |
| chosen = [grouped[lengths[0]], grouped[lengths[-1]]] | |
| if len(lengths) > 2: | |
| chosen.insert(1, grouped[lengths[len(lengths) // 2]]) | |
| return chosen | |
| def compare_export_outputs(reference, outputs, attention_mask, decoder: dict[str, object]) -> dict[str, object]: | |
| """Compare active-token numerics and decoded spans for a padded text batch.""" | |
| import numpy as np | |
| reference = np.asarray(reference) | |
| outputs = np.asarray(outputs) | |
| attention_mask = np.asarray(attention_mask) | |
| if reference.ndim != 4 or outputs.ndim != 4 or reference.shape != outputs.shape: | |
| raise ValueError("PyTorch/ONNX logits must match as [batch, sequence, category, bilou]") | |
| if attention_mask.shape != reference.shape[:2] or not np.isin(attention_mask, [0, 1]).all(): | |
| raise ValueError("parity attention mask must match the batch/sequence axes") | |
| if np.any(np.diff(attention_mask, axis=1) > 0): | |
| raise ValueError("parity inputs must use right padding") | |
| if reference.shape[2] != len(decoder["categories"]) or reference.shape[3] != len(BILOU_LABELS): | |
| raise ValueError("PyTorch logits category or BILOU axis differs from decoder metadata") | |
| active = attention_mask.astype(bool) | |
| if not active.any(): | |
| raise ValueError("parity batch contains no attended tokens") | |
| if len(set(int(row.sum()) for row in active)) < 2 or active.all(): | |
| raise ValueError("parity batch must exercise mixed attended lengths and padding") | |
| difference = np.abs(reference - outputs)[active] | |
| argmax_equal = np.argmax(reference, axis=-1)[active] == np.argmax(outputs, axis=-1)[active] | |
| by_case = [] | |
| for index in range(reference.shape[0]): | |
| token_count = int(active[index].sum()) | |
| ref_spans = decoded_span_signature(reference[index:index + 1, :token_count], decoder) | |
| onnx_spans = decoded_span_signature(outputs[index:index + 1, :token_count], decoder) | |
| span_keys = lambda rows: [(row["category"], row["start_token"], row["end_token"], tuple(row["labels"])) for row in rows] | |
| confidence_diffs = ( | |
| [abs(float(left["confidence"]) - float(right["confidence"])) for left, right in zip(ref_spans, onnx_spans)] | |
| if span_keys(ref_spans) == span_keys(onnx_spans) | |
| else [] | |
| ) | |
| by_case.append({"attended_tokens": token_count, | |
| "decoded_span_parity": span_keys(ref_spans) == span_keys(onnx_spans), | |
| "decoded_span_confidence_max_difference": round(max(confidence_diffs), 6) if confidence_diffs else None, | |
| "reference_decoded_spans": ref_spans, | |
| "onnx_decoded_spans": onnx_spans}) | |
| return { | |
| "max_abs_logit_difference": round(float(difference.max()), 6), | |
| "mean_abs_logit_difference": round(float(difference.mean()), 6), | |
| "argmax_agreement": round(float(argmax_equal.mean()), 6), | |
| "decoded_span_parity": all(row["decoded_span_parity"] for row in by_case), | |
| "parity_case_count": len(by_case), | |
| "padded_sequence_length": int(reference.shape[1]), | |
| "cases": by_case, | |
| } | |
| def decoded_span_signature(logits, decoder: dict[str, object]) -> list[dict[str, object]]: | |
| """Apply the exact exported decoder to a single batch of model logits.""" | |
| from sponsor_detection.model.smart_segment_decoder import decode_multi_head, softmax_probabilities | |
| import numpy as np | |
| values = np.asarray(logits) | |
| if values.ndim != 4 or values.shape[0] != 1: | |
| raise ValueError("decoded parity expects [1, sequence, category, bilou] logits") | |
| if values.shape[2] != len(decoder["categories"]) or values.shape[3] != len(BILOU_LABELS): | |
| raise ValueError("decoded parity logits do not match declared axes") | |
| spans = decode_multi_head( | |
| softmax_probabilities(values[0]), | |
| categories=decoder["categories"], | |
| thresholds=decoder["thresholds"], | |
| default_threshold=decoder["default_threshold"], | |
| enabled_categories=decoder["enabled_categories"], | |
| labels=decoder["labels"], | |
| ) | |
| return [ | |
| {"category": span.category, "start_token": span.start_index, | |
| "end_token": span.end_index, "confidence": span.confidence, | |
| "labels": list(span.labels)} | |
| for span in spans | |
| ] | |
| def export_multi_head_from_config(config_path: Path) -> dict[str, object]: | |
| """Export the trained multi-head model to a versioned, hash-pinned bundle.""" | |
| import onnx | |
| with config_path.open("rb") as source: | |
| configuration = tomllib.load(source) | |
| export = configuration["export"] | |
| release = configuration["release"] | |
| checkpoint_directory = Path(export["checkpoint_directory"]) | |
| output_directory = Path(export["output_directory"]) | |
| if output_directory.exists() and any(output_directory.iterdir()): | |
| raise FileExistsError(f"refusing to overwrite non-empty export directory: {output_directory}") | |
| parity_inputs, parity_input_record = _load_development_parity_texts(export) | |
| revision = str(export["encoder_revision"]) | |
| sequence_length = int(export.get("sequence_length", 256)) | |
| opset_version = int(export.get("opset_version", 17)) | |
| if sequence_length < 2 or opset_version < 1: | |
| raise ValueError("export sequence_length and opset_version must be positive") | |
| model = load_saved_multi_head( | |
| checkpoint_directory, | |
| revision=revision, | |
| allow_legacy_architecture_fallback=bool( | |
| export.get("allow_legacy_architecture_fallback", False) | |
| ), | |
| ) | |
| categories = list(model.categories) | |
| if categories != ["sponsor", "selfpromo", "interaction"]: | |
| raise ValueError("experimental schema v2 requires category order sponsor/selfpromo/interaction") | |
| decoder = decoder_configuration(model, export) | |
| source_checkpoint = { | |
| "path": str(checkpoint_directory), | |
| "model_sha256": sha256_file(checkpoint_directory / "model.safetensors"), | |
| "config_sha256": sha256_file(checkpoint_directory / "config.json"), | |
| } | |
| if not (checkpoint_directory / "tokenizer.json").is_file(): | |
| raise ValueError("checkpoint export requires tokenizer.json to pin tokenizer identity") | |
| tokenizer = _load_tokenizer(checkpoint_directory) | |
| parity_inputs = _mixed_length_parity_texts(tokenizer, parity_inputs, max_length=sequence_length) | |
| output_directory.mkdir(parents=True, exist_ok=True) | |
| fp32_path = output_directory / "smart_segments.fp32.onnx" | |
| int8_path = output_directory / "smart_segments.int8.onnx" | |
| _export_fp32( | |
| model, fp32_path, sequence_length=sequence_length, opset_version=opset_version | |
| ) | |
| onnx.checker.check_model(str(fp32_path)) | |
| _quantize(fp32_path, int8_path, export.get("quantize_op_types", ["Gather"])) | |
| onnx.checker.check_model(str(int8_path)) | |
| tokenizer_path = output_directory / "tokenizer.json" | |
| import shutil | |
| copied_tokenizer_assets = [] | |
| for name in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "added_tokens.json", "vocab.json", "merges.txt", "spiece.model", "sentencepiece.bpe.model", "tokenizer.model"): | |
| source = checkpoint_directory / name | |
| if source.is_file(): | |
| shutil.copy2(source, output_directory / name) | |
| copied_tokenizer_assets.append(name) | |
| if "tokenizer.json" not in copied_tokenizer_assets: | |
| raise ValueError("checkpoint export requires tokenizer.json to pin tokenizer identity") | |
| fp32_parity = _parity( | |
| model, fp32_path, tokenizer, parity_inputs, max_length=sequence_length, decoder=decoder | |
| ) | |
| int8_parity = _parity( | |
| model, int8_path, tokenizer, parity_inputs, max_length=sequence_length, decoder=decoder | |
| ) | |
| if not fp32_parity["decoded_span_parity"] or not int8_parity["decoded_span_parity"]: | |
| raise ValueError("exported model decoded spans differ from PyTorch at the pinned decoder settings") | |
| decoder_path = output_directory / "decoder_config.json" | |
| decoder_path.write_text(json.dumps(decoder, indent=2, sort_keys=True) + "\n", encoding="utf-8") | |
| tokenizer_assets = { | |
| name: {"file": name, "sha256": sha256_file(output_directory / name)} | |
| for name in copied_tokenizer_assets | |
| } | |
| manifest = { | |
| "schema_version": MODEL_MANIFEST_SCHEMA_VERSION, | |
| "artifact_profile": "experimental_multi_head_v2", | |
| "model_version": str(release["model_version"]), | |
| "source_checkpoint": source_checkpoint, | |
| "encoder_revision": revision, | |
| "release_eligible": False, | |
| "automatic_actions_enabled": False, | |
| "android_compatible": False, | |
| "runtime_compatibility": "research_onnx_only_rank4_not_legacy_sponsor_rank3", | |
| "model": {"file": int8_path.name, "sha256": sha256_file(int8_path)}, | |
| "tokenizer": {"file": tokenizer_path.name, "sha256": sha256_file(tokenizer_path)}, | |
| "tokenizer_assets": tokenizer_assets, | |
| "inputs": { | |
| "input_ids": {"dtype": "int64", "shape": ["batch", "sequence"]}, | |
| "attention_mask": {"dtype": "int64", "shape": ["batch", "sequence"]}, | |
| }, | |
| "outputs": { | |
| "segment_logits": { | |
| "dtype": "float32", | |
| "shape": ["batch", "sequence", "category", "bilou"], | |
| } | |
| }, | |
| "categories": {str(index): category for index, category in enumerate(categories)}, | |
| "enabled_categories": list(decoder["enabled_categories"]), | |
| "trained_categories": list(decoder["trained_categories"]), | |
| "disabled_categories": [category for category in categories if category not in decoder["enabled_categories"]], | |
| "labels": {"0": "O", "1": "B", "2": "I", "3": "L", "4": "U"}, | |
| "normalization_version": 1, | |
| "decoder_version": 2, | |
| "decoder_config_sha256": sha256_file(decoder_path), | |
| "decoder": { | |
| "default_threshold": decoder["default_threshold"], | |
| "thresholds": decoder["thresholds"], | |
| "merge_gap_ms": decoder["merge_gap_ms"], | |
| "min_span_chars": decoder["min_span_chars"], | |
| "enabled_categories": list(decoder["enabled_categories"]), | |
| "category_order": categories, | |
| "label_order": list(BILOU_LABELS), | |
| }, | |
| "minimum_android_runtime_version": 1, | |
| } | |
| manifest_path = output_directory / "manifest.json" | |
| write_json_atomic(manifest_path, manifest) | |
| parsed = parse_model_manifest(json.loads(manifest_path.read_text(encoding="utf-8"))) | |
| verified = verify_artifacts(parsed, output_directory) | |
| verified["decoder_config"] = sha256_file(decoder_path) | |
| for name, asset in tokenizer_assets.items(): | |
| if sha256_file(output_directory / asset["file"]) != asset["sha256"]: | |
| raise ValueError(f"tokenizer asset hash mismatch: {name}") | |
| verified[f"tokenizer_asset:{name}"] = asset["sha256"] | |
| interface = int8_parity["onnx_interface"] | |
| report = { | |
| "model_version": manifest["model_version"], | |
| "source_checkpoint": source_checkpoint, | |
| "encoder_revision": revision, | |
| "artifact_profile": manifest["artifact_profile"], | |
| "release_eligible": False, | |
| "automatic_actions_enabled": False, | |
| "android_compatible": False, | |
| "runtime_compatibility": manifest["runtime_compatibility"], | |
| "categories": categories, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "artifacts": { | |
| "fp32": { | |
| "path": str(fp32_path), | |
| "bytes": fp32_path.stat().st_size, | |
| "sha256": sha256_file(fp32_path), | |
| }, | |
| "int8": { | |
| "path": str(int8_path), | |
| "bytes": int8_path.stat().st_size, | |
| "sha256": sha256_file(int8_path), | |
| }, | |
| "manifest": str(manifest_path), | |
| }, | |
| "verified_artifacts": verified, | |
| "onnx_interface": interface, | |
| "parity": {"fp32": fp32_parity, "int8": int8_parity}, | |
| "parity_input_example_count": len(parity_inputs), | |
| "parity_input": parity_input_record, | |
| "decoder": decoder, | |
| } | |
| write_json_atomic(output_directory / "export_report.json", report) | |
| return report | |
| def _load_tokenizer(checkpoint_directory: Path): | |
| from transformers import AutoTokenizer | |
| return AutoTokenizer.from_pretrained(str(checkpoint_directory)) | |
| def _load_development_parity_texts(export: dict[str, object]) -> tuple[list[str], dict[str, object]]: | |
| """Load only hash-pinned development text explicitly named by export config.""" | |
| import re | |
| if export.get("parity_input_role") != "development": | |
| raise ValueError("export parity_input_role must explicitly be development") | |
| path_value = export.get("parity_input_path") | |
| expected_hash = export.get("parity_input_sha256") | |
| column = str(export.get("parity_input_text_column", "text")) | |
| if not isinstance(path_value, str) or not path_value: | |
| raise ValueError("export requires parity_input_path") | |
| if not isinstance(expected_hash, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_hash): | |
| raise ValueError("export requires a declared lowercase parity_input_sha256") | |
| path = Path(path_value) | |
| actual_hash = sha256_file(path) | |
| if actual_hash != expected_hash: | |
| raise ValueError("development parity input hash differs from the configured hash") | |
| if path.suffix == ".parquet": | |
| import pyarrow.parquet as parquet | |
| table = parquet.read_table(path, columns=[column]).slice(0, 64) | |
| raw = table.column(column).to_pylist() | |
| elif path.suffix == ".jsonl": | |
| raw = [json.loads(line)[column] for line in path.read_text(encoding="utf-8").splitlines()[:64] if line.strip()] | |
| else: | |
| raise ValueError("development parity input must be a .parquet or .jsonl file") | |
| texts = [value for value in raw if isinstance(value, str) and value.strip()] | |
| if len(texts) < 2: | |
| raise ValueError("development parity input must contain at least two non-empty text examples") | |
| return texts, {"role": "development", "path": path_value, "sha256": actual_hash, | |
| "text_column": column, "candidate_example_count": len(texts)} | |