CuriousDragon's picture
Publish verified Flow Smart Segments curated-category-encoder-ensemble-20261004
8fcf7c9 verified
Raw History Blame Contribute Delete
14.6 kB
from __future__ import annotations
import importlib.metadata
import json
import shutil
import tomllib
from pathlib import Path
from sponsor_detection.data.profile import sha256_file, write_json_atomic
from sponsor_detection.inference.onnx_pipeline import OnnxFullTranscriptSponsorDetector
from sponsor_detection.inference.windowing import normalize_cue_text
from sponsor_detection.video_evaluation import (
evaluate_full_videos_with_detector,
load_frozen_benchmark,
)
TOKENIZER_GOLDEN_INPUTS = (
"This episode is brought to you by Acme.",
"Visit https://example.com/deal and use code FLOW20.",
"Save 20% before 12:30 today!",
"I’m testing contractions: we're, you've, they'll.",
"Café naïve résumé — Unicode stays deterministic.",
"Emoji boundary 🚀 sponsor message 🎧 end.",
"Multiple spaces\tand\nnewlines are normalized.",
)
def compare_evaluation_reports(
baseline: dict[str, object], candidate: dict[str, object]
) -> dict[str, object]:
baseline_by_id = {record["video_id"]: record for record in baseline["videos"]}
candidate_by_id = {record["video_id"]: record for record in candidate["videos"]}
if baseline_by_id.keys() != candidate_by_id.keys():
raise ValueError("evaluation reports contain different video sets")
differing_videos = []
interval_matches = 0
presence_matches = 0
for video_id, baseline_record in baseline_by_id.items():
candidate_record = candidate_by_id[video_id]
baseline_intervals = [
(span["start_ms"], span["end_ms"])
for span in baseline_record["predicted_spans"]
]
candidate_intervals = [
(span["start_ms"], span["end_ms"])
for span in candidate_record["predicted_spans"]
]
same_intervals = baseline_intervals == candidate_intervals
same_presence = bool(baseline_intervals) == bool(candidate_intervals)
interval_matches += int(same_intervals)
presence_matches += int(same_presence)
if not same_intervals:
differing_videos.append(
{
"video_id": video_id,
"baseline": baseline_intervals,
"candidate": candidate_intervals,
}
)
total = len(baseline_by_id)
return {
"videos": total,
"exact_interval_matches": interval_matches,
"presence_matches": presence_matches,
"exact_interval_match_rate": interval_matches / total if total else 0.0,
"presence_match_rate": presence_matches / total if total else 0.0,
"differing_videos": differing_videos,
}
def _export_fp32(checkpoint_path: Path, output_path: Path, *, opset: int) -> None:
import onnx
import torch
from transformers import AutoModelForTokenClassification
class LogitsOnly(torch.nn.Module):
def __init__(self, model) -> None:
super().__init__()
self.model = model
def forward(self, input_ids, attention_mask):
return self.model(
input_ids=input_ids, attention_mask=attention_mask
).logits
model = LogitsOnly(
AutoModelForTokenClassification.from_pretrained(checkpoint_path)
).eval()
input_ids = torch.tensor([[50281, 100, 101, 50282]], dtype=torch.long)
attention_mask = torch.ones_like(input_ids)
torch.onnx.export(
model,
(input_ids, attention_mask),
output_path,
input_names=["input_ids", "attention_mask"],
output_names=["logits"],
dynamic_shapes=(
{0: "batch", 1: "sequence"},
{0: "batch", 1: "sequence"},
),
opset_version=opset,
dynamo=True,
external_data=False,
)
onnx.checker.check_model(onnx.load(output_path))
def _quantize(fp32_path: Path, int8_path: Path) -> None:
from onnxruntime.quantization import QuantType, quantize_dynamic
quantize_dynamic(
fp32_path,
int8_path,
weight_type=QuantType.QInt8,
per_channel=True,
reduce_range=False,
op_types_to_quantize=["Gather"],
)
def _convert_to_ort(int8_path: Path) -> tuple[Path, Path]:
from onnxruntime.tools.convert_onnx_models_to_ort import (
OptimizationStyle,
convert_onnx_models_to_ort,
)
convert_onnx_models_to_ort(
int8_path,
output_dir=int8_path.parent,
optimization_styles=[OptimizationStyle.Fixed],
enable_type_reduction=True,
)
ort_path = int8_path.with_suffix(".ort")
operator_config_path = int8_path.with_name(
f"{int8_path.stem}.required_operators_and_types.config"
)
if not ort_path.is_file() or not operator_config_path.is_file():
raise RuntimeError("ONNX Runtime did not produce the expected Android artifacts")
return ort_path, operator_config_path
def _write_tokenizer_goldens(checkpoint_path: Path, output_path: Path) -> None:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
records = []
for text in TOKENIZER_GOLDEN_INPUTS:
normalized = normalize_cue_text(text)
encoded = tokenizer(
normalized,
add_special_tokens=True,
return_offsets_mapping=True,
)
records.append(
{
"input": text,
"normalized": normalized,
"input_ids": [int(value) for value in encoded["input_ids"]],
"attention_mask": [
int(value) for value in encoded["attention_mask"]
],
"offset_mapping": [
[int(start), int(end)]
for start, end in encoded["offset_mapping"]
],
}
)
write_json_atomic(output_path, {"schema_version": 1, "cases": records})
def _artifact(path: Path) -> dict[str, object]:
return {
"path": str(path),
"bytes": path.stat().st_size,
"sha256": sha256_file(path),
}
def export_android_bundle_from_config(
path: Path, *, progress_callback=None
) -> dict[str, object]:
with path.open("rb") as source:
configuration = tomllib.load(source)["android_export"]
checkpoint_path = Path(configuration["checkpoint_path"])
benchmark_path = Path(configuration["benchmark_path"])
benchmark_manifest_path = Path(configuration["benchmark_manifest_path"])
baseline_report_path = Path(configuration["baseline_report_path"])
output_directory = Path(configuration["output_directory"])
report_path = Path(configuration["report_path"])
feedback_schema_path = Path(configuration["feedback_schema_path"])
confidence_threshold = float(configuration.get("confidence_threshold", 0.0))
max_length = int(configuration.get("max_length", 768))
overlap_tokens = int(configuration.get("overlap_tokens", 128))
batch_size = int(configuration.get("batch_size", 16))
merge_gap_characters = int(configuration.get("merge_gap_characters", 24))
merge_gap_ms = int(configuration.get("merge_gap_ms", 1500))
maximum_f1_drop = float(configuration.get("maximum_span_f1_drop", 0.01))
artifact_stem = str(configuration.get("artifact_stem", "sponsor_detector_v1"))
package_name = str(
configuration.get("package_name", "ettin-17m-sponsor-v1-android-int8")
)
output_directory.mkdir(parents=True, exist_ok=True)
android_directory = output_directory / "android"
android_directory.mkdir(parents=True, exist_ok=True)
fp32_path = output_directory / f"{artifact_stem}.fp32.onnx"
int8_path = output_directory / f"{artifact_stem}.int8.onnx"
if progress_callback:
progress_callback("export_fp32")
_export_fp32(checkpoint_path, fp32_path, opset=int(configuration.get("opset", 18)))
if progress_callback:
progress_callback("quantize_int8")
_quantize(fp32_path, int8_path)
if progress_callback:
progress_callback("convert_ort")
ort_path, operator_config_path = _convert_to_ort(int8_path)
bundled_model_path = android_directory / f"{artifact_stem}.int8.ort"
bundled_operator_config_path = android_directory / "required_operators_and_types.config"
shutil.copyfile(ort_path, bundled_model_path)
shutil.copyfile(operator_config_path, bundled_operator_config_path)
for name in ("tokenizer.json", "tokenizer_config.json", "config.json"):
shutil.copyfile(checkpoint_path / name, android_directory / name)
tokenizer_goldens_path = android_directory / "tokenizer_goldens.json"
_write_tokenizer_goldens(checkpoint_path, tokenizer_goldens_path)
bundled_feedback_schema_path = android_directory / "sponsor_feedback_v1.schema.json"
shutil.copyfile(feedback_schema_path, bundled_feedback_schema_path)
transcript_paths, expected_by_video, dataset_provenance = load_frozen_benchmark(
benchmark_path, benchmark_manifest_path
)
baseline = json.loads(baseline_report_path.read_text(encoding="utf-8"))
if baseline["dataset"]["benchmark_sha256"] != dataset_provenance["benchmark_sha256"]:
raise ValueError("baseline report does not match the frozen benchmark")
checkpoint_sha256 = sha256_file(checkpoint_path / "model.safetensors")
if baseline["checkpoint"]["model_sha256"] != checkpoint_sha256:
raise ValueError("baseline report does not match the v1 checkpoint")
evaluation_reports = {}
for name, model_path in (("fp32", fp32_path), ("int8", bundled_model_path)):
if progress_callback:
progress_callback(f"evaluate_{name}")
detector = OnnxFullTranscriptSponsorDetector(
model_path,
checkpoint_path,
confidence_threshold=confidence_threshold,
max_length=max_length,
overlap_tokens=overlap_tokens,
batch_size=batch_size,
merge_gap_characters=merge_gap_characters,
merge_gap_ms=merge_gap_ms,
)
evaluation_path = report_path.with_name(
f"{report_path.stem}_{name}_evaluation.json"
)
evaluation_reports[name] = evaluate_full_videos_with_detector(
transcript_paths,
expected_by_video,
detector,
dataset_provenance,
checkpoint_provenance={
"path": str(model_path),
"model_sha256": sha256_file(model_path),
"source_checkpoint_sha256": checkpoint_sha256,
},
decoder_config_path=None,
report_path=evaluation_path,
progress_callback=None,
)
fp32_parity = compare_evaluation_reports(baseline, evaluation_reports["fp32"])
int8_parity = compare_evaluation_reports(baseline, evaluation_reports["int8"])
baseline_f1 = float(baseline["span_metrics_by_temporal_iou"]["0.5"]["f1"])
int8_f1 = float(
evaluation_reports["int8"]["span_metrics_by_temporal_iou"]["0.5"]["f1"]
)
baseline_recall = float(baseline["video_presence"]["recall"])
int8_recall = float(evaluation_reports["int8"]["video_presence"]["recall"])
baseline_false_positives = int(baseline["video_presence"].get("false_positive", 0))
int8_false_positives = int(
evaluation_reports["int8"]["video_presence"].get("false_positive", 0)
)
gates = {
"fp32_presence_exact": fp32_parity["presence_matches"] == fp32_parity["videos"],
"fp32_intervals_exact": fp32_parity["exact_interval_matches"]
== fp32_parity["videos"],
"int8_span_f1_within_limit": baseline_f1 - int8_f1 <= maximum_f1_drop,
"int8_video_recall_not_lower": int8_recall >= baseline_recall,
"int8_false_positives_not_higher": int8_false_positives
<= baseline_false_positives,
"int8_presence_exact": int8_parity["presence_matches"]
== int8_parity["videos"],
}
status = "validated" if all(gates.values()) else "failed"
package_files = {
"model": _artifact(bundled_model_path),
"operator_config": _artifact(bundled_operator_config_path),
"tokenizer": _artifact(android_directory / "tokenizer.json"),
"tokenizer_config": _artifact(android_directory / "tokenizer_config.json"),
"model_config": _artifact(android_directory / "config.json"),
"tokenizer_goldens": _artifact(tokenizer_goldens_path),
"feedback_schema": _artifact(bundled_feedback_schema_path),
}
manifest = {
"schema_version": 1,
"name": package_name,
"status": status,
"source": {
"checkpoint_path": str(checkpoint_path),
"checkpoint_sha256": checkpoint_sha256,
"base_model": "jhu-clsp/ettin-encoder-17m",
"base_model_license": "MIT",
},
"runtime": {
"format": "ORT",
"execution_provider": "CPUExecutionProvider",
"quantization": "embedding_int8_per_channel",
"opset": int(configuration.get("opset", 18)),
},
"inference": {
"labels": ["O", "B-SPONSOR", "I-SPONSOR", "L-SPONSOR", "U-SPONSOR"],
"confidence_threshold": confidence_threshold,
"max_length": max_length,
"overlap_tokens": overlap_tokens,
"merge_gap_characters": merge_gap_characters,
"merge_gap_ms": merge_gap_ms,
},
"package": {
"directory": str(android_directory),
"bytes": sum(int(artifact["bytes"]) for artifact in package_files.values()),
"files": package_files,
},
"validation": {
"benchmark_sha256": dataset_provenance["benchmark_sha256"],
"baseline_report": _artifact(baseline_report_path),
"fp32_parity": fp32_parity,
"int8_parity": int8_parity,
"baseline_span_f1_at_iou_0.5": baseline_f1,
"int8_span_f1_at_iou_0.5": int8_f1,
"maximum_allowed_span_f1_drop": maximum_f1_drop,
"gates": gates,
"fp32_evaluation": evaluation_reports["fp32"],
"int8_evaluation": evaluation_reports["int8"],
},
"toolchain": {
package: importlib.metadata.version(package)
for package in (
"torch",
"transformers",
"onnx",
"onnxruntime",
"onnxscript",
)
},
}
write_json_atomic(android_directory / "manifest.json", manifest)
write_json_atomic(report_path, manifest)
return manifest