Download code/src/sponsor_detection/android_export.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 14.6 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/android_export.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/src/sponsor_detection/android_export.py
-
curl -L -o android_export.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/android_export.py
14.6 kB
| from __future__ import annotations | |
| import importlib.metadata | |
| import json | |
| import shutil | |
| import tomllib | |
| from pathlib import Path | |
| from sponsor_detection.data.profile import sha256_file, write_json_atomic | |
| from sponsor_detection.inference.onnx_pipeline import OnnxFullTranscriptSponsorDetector | |
| from sponsor_detection.inference.windowing import normalize_cue_text | |
| from sponsor_detection.video_evaluation import ( | |
| evaluate_full_videos_with_detector, | |
| load_frozen_benchmark, | |
| ) | |
| TOKENIZER_GOLDEN_INPUTS = ( | |
| "This episode is brought to you by Acme.", | |
| "Visit https://example.com/deal and use code FLOW20.", | |
| "Save 20% before 12:30 today!", | |
| "I’m testing contractions: we're, you've, they'll.", | |
| "Café naïve résumé — Unicode stays deterministic.", | |
| "Emoji boundary 🚀 sponsor message 🎧 end.", | |
| "Multiple spaces\tand\nnewlines are normalized.", | |
| ) | |
| def compare_evaluation_reports( | |
| baseline: dict[str, object], candidate: dict[str, object] | |
| ) -> dict[str, object]: | |
| baseline_by_id = {record["video_id"]: record for record in baseline["videos"]} | |
| candidate_by_id = {record["video_id"]: record for record in candidate["videos"]} | |
| if baseline_by_id.keys() != candidate_by_id.keys(): | |
| raise ValueError("evaluation reports contain different video sets") | |
| differing_videos = [] | |
| interval_matches = 0 | |
| presence_matches = 0 | |
| for video_id, baseline_record in baseline_by_id.items(): | |
| candidate_record = candidate_by_id[video_id] | |
| baseline_intervals = [ | |
| (span["start_ms"], span["end_ms"]) | |
| for span in baseline_record["predicted_spans"] | |
| ] | |
| candidate_intervals = [ | |
| (span["start_ms"], span["end_ms"]) | |
| for span in candidate_record["predicted_spans"] | |
| ] | |
| same_intervals = baseline_intervals == candidate_intervals | |
| same_presence = bool(baseline_intervals) == bool(candidate_intervals) | |
| interval_matches += int(same_intervals) | |
| presence_matches += int(same_presence) | |
| if not same_intervals: | |
| differing_videos.append( | |
| { | |
| "video_id": video_id, | |
| "baseline": baseline_intervals, | |
| "candidate": candidate_intervals, | |
| } | |
| ) | |
| total = len(baseline_by_id) | |
| return { | |
| "videos": total, | |
| "exact_interval_matches": interval_matches, | |
| "presence_matches": presence_matches, | |
| "exact_interval_match_rate": interval_matches / total if total else 0.0, | |
| "presence_match_rate": presence_matches / total if total else 0.0, | |
| "differing_videos": differing_videos, | |
| } | |
| def _export_fp32(checkpoint_path: Path, output_path: Path, *, opset: int) -> None: | |
| import onnx | |
| import torch | |
| from transformers import AutoModelForTokenClassification | |
| class LogitsOnly(torch.nn.Module): | |
| def __init__(self, model) -> None: | |
| super().__init__() | |
| self.model = model | |
| def forward(self, input_ids, attention_mask): | |
| return self.model( | |
| input_ids=input_ids, attention_mask=attention_mask | |
| ).logits | |
| model = LogitsOnly( | |
| AutoModelForTokenClassification.from_pretrained(checkpoint_path) | |
| ).eval() | |
| input_ids = torch.tensor([[50281, 100, 101, 50282]], dtype=torch.long) | |
| attention_mask = torch.ones_like(input_ids) | |
| torch.onnx.export( | |
| model, | |
| (input_ids, attention_mask), | |
| output_path, | |
| input_names=["input_ids", "attention_mask"], | |
| output_names=["logits"], | |
| dynamic_shapes=( | |
| {0: "batch", 1: "sequence"}, | |
| {0: "batch", 1: "sequence"}, | |
| ), | |
| opset_version=opset, | |
| dynamo=True, | |
| external_data=False, | |
| ) | |
| onnx.checker.check_model(onnx.load(output_path)) | |
| def _quantize(fp32_path: Path, int8_path: Path) -> None: | |
| from onnxruntime.quantization import QuantType, quantize_dynamic | |
| quantize_dynamic( | |
| fp32_path, | |
| int8_path, | |
| weight_type=QuantType.QInt8, | |
| per_channel=True, | |
| reduce_range=False, | |
| op_types_to_quantize=["Gather"], | |
| ) | |
| def _convert_to_ort(int8_path: Path) -> tuple[Path, Path]: | |
| from onnxruntime.tools.convert_onnx_models_to_ort import ( | |
| OptimizationStyle, | |
| convert_onnx_models_to_ort, | |
| ) | |
| convert_onnx_models_to_ort( | |
| int8_path, | |
| output_dir=int8_path.parent, | |
| optimization_styles=[OptimizationStyle.Fixed], | |
| enable_type_reduction=True, | |
| ) | |
| ort_path = int8_path.with_suffix(".ort") | |
| operator_config_path = int8_path.with_name( | |
| f"{int8_path.stem}.required_operators_and_types.config" | |
| ) | |
| if not ort_path.is_file() or not operator_config_path.is_file(): | |
| raise RuntimeError("ONNX Runtime did not produce the expected Android artifacts") | |
| return ort_path, operator_config_path | |
| def _write_tokenizer_goldens(checkpoint_path: Path, output_path: Path) -> None: | |
| from transformers import AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained(checkpoint_path) | |
| records = [] | |
| for text in TOKENIZER_GOLDEN_INPUTS: | |
| normalized = normalize_cue_text(text) | |
| encoded = tokenizer( | |
| normalized, | |
| add_special_tokens=True, | |
| return_offsets_mapping=True, | |
| ) | |
| records.append( | |
| { | |
| "input": text, | |
| "normalized": normalized, | |
| "input_ids": [int(value) for value in encoded["input_ids"]], | |
| "attention_mask": [ | |
| int(value) for value in encoded["attention_mask"] | |
| ], | |
| "offset_mapping": [ | |
| [int(start), int(end)] | |
| for start, end in encoded["offset_mapping"] | |
| ], | |
| } | |
| ) | |
| write_json_atomic(output_path, {"schema_version": 1, "cases": records}) | |
| def _artifact(path: Path) -> dict[str, object]: | |
| return { | |
| "path": str(path), | |
| "bytes": path.stat().st_size, | |
| "sha256": sha256_file(path), | |
| } | |
| def export_android_bundle_from_config( | |
| path: Path, *, progress_callback=None | |
| ) -> dict[str, object]: | |
| with path.open("rb") as source: | |
| configuration = tomllib.load(source)["android_export"] | |
| checkpoint_path = Path(configuration["checkpoint_path"]) | |
| benchmark_path = Path(configuration["benchmark_path"]) | |
| benchmark_manifest_path = Path(configuration["benchmark_manifest_path"]) | |
| baseline_report_path = Path(configuration["baseline_report_path"]) | |
| output_directory = Path(configuration["output_directory"]) | |
| report_path = Path(configuration["report_path"]) | |
| feedback_schema_path = Path(configuration["feedback_schema_path"]) | |
| confidence_threshold = float(configuration.get("confidence_threshold", 0.0)) | |
| max_length = int(configuration.get("max_length", 768)) | |
| overlap_tokens = int(configuration.get("overlap_tokens", 128)) | |
| batch_size = int(configuration.get("batch_size", 16)) | |
| merge_gap_characters = int(configuration.get("merge_gap_characters", 24)) | |
| merge_gap_ms = int(configuration.get("merge_gap_ms", 1500)) | |
| maximum_f1_drop = float(configuration.get("maximum_span_f1_drop", 0.01)) | |
| artifact_stem = str(configuration.get("artifact_stem", "sponsor_detector_v1")) | |
| package_name = str( | |
| configuration.get("package_name", "ettin-17m-sponsor-v1-android-int8") | |
| ) | |
| output_directory.mkdir(parents=True, exist_ok=True) | |
| android_directory = output_directory / "android" | |
| android_directory.mkdir(parents=True, exist_ok=True) | |
| fp32_path = output_directory / f"{artifact_stem}.fp32.onnx" | |
| int8_path = output_directory / f"{artifact_stem}.int8.onnx" | |
| if progress_callback: | |
| progress_callback("export_fp32") | |
| _export_fp32(checkpoint_path, fp32_path, opset=int(configuration.get("opset", 18))) | |
| if progress_callback: | |
| progress_callback("quantize_int8") | |
| _quantize(fp32_path, int8_path) | |
| if progress_callback: | |
| progress_callback("convert_ort") | |
| ort_path, operator_config_path = _convert_to_ort(int8_path) | |
| bundled_model_path = android_directory / f"{artifact_stem}.int8.ort" | |
| bundled_operator_config_path = android_directory / "required_operators_and_types.config" | |
| shutil.copyfile(ort_path, bundled_model_path) | |
| shutil.copyfile(operator_config_path, bundled_operator_config_path) | |
| for name in ("tokenizer.json", "tokenizer_config.json", "config.json"): | |
| shutil.copyfile(checkpoint_path / name, android_directory / name) | |
| tokenizer_goldens_path = android_directory / "tokenizer_goldens.json" | |
| _write_tokenizer_goldens(checkpoint_path, tokenizer_goldens_path) | |
| bundled_feedback_schema_path = android_directory / "sponsor_feedback_v1.schema.json" | |
| shutil.copyfile(feedback_schema_path, bundled_feedback_schema_path) | |
| transcript_paths, expected_by_video, dataset_provenance = load_frozen_benchmark( | |
| benchmark_path, benchmark_manifest_path | |
| ) | |
| baseline = json.loads(baseline_report_path.read_text(encoding="utf-8")) | |
| if baseline["dataset"]["benchmark_sha256"] != dataset_provenance["benchmark_sha256"]: | |
| raise ValueError("baseline report does not match the frozen benchmark") | |
| checkpoint_sha256 = sha256_file(checkpoint_path / "model.safetensors") | |
| if baseline["checkpoint"]["model_sha256"] != checkpoint_sha256: | |
| raise ValueError("baseline report does not match the v1 checkpoint") | |
| evaluation_reports = {} | |
| for name, model_path in (("fp32", fp32_path), ("int8", bundled_model_path)): | |
| if progress_callback: | |
| progress_callback(f"evaluate_{name}") | |
| detector = OnnxFullTranscriptSponsorDetector( | |
| model_path, | |
| checkpoint_path, | |
| confidence_threshold=confidence_threshold, | |
| max_length=max_length, | |
| overlap_tokens=overlap_tokens, | |
| batch_size=batch_size, | |
| merge_gap_characters=merge_gap_characters, | |
| merge_gap_ms=merge_gap_ms, | |
| ) | |
| evaluation_path = report_path.with_name( | |
| f"{report_path.stem}_{name}_evaluation.json" | |
| ) | |
| evaluation_reports[name] = evaluate_full_videos_with_detector( | |
| transcript_paths, | |
| expected_by_video, | |
| detector, | |
| dataset_provenance, | |
| checkpoint_provenance={ | |
| "path": str(model_path), | |
| "model_sha256": sha256_file(model_path), | |
| "source_checkpoint_sha256": checkpoint_sha256, | |
| }, | |
| decoder_config_path=None, | |
| report_path=evaluation_path, | |
| progress_callback=None, | |
| ) | |
| fp32_parity = compare_evaluation_reports(baseline, evaluation_reports["fp32"]) | |
| int8_parity = compare_evaluation_reports(baseline, evaluation_reports["int8"]) | |
| baseline_f1 = float(baseline["span_metrics_by_temporal_iou"]["0.5"]["f1"]) | |
| int8_f1 = float( | |
| evaluation_reports["int8"]["span_metrics_by_temporal_iou"]["0.5"]["f1"] | |
| ) | |
| baseline_recall = float(baseline["video_presence"]["recall"]) | |
| int8_recall = float(evaluation_reports["int8"]["video_presence"]["recall"]) | |
| baseline_false_positives = int(baseline["video_presence"].get("false_positive", 0)) | |
| int8_false_positives = int( | |
| evaluation_reports["int8"]["video_presence"].get("false_positive", 0) | |
| ) | |
| gates = { | |
| "fp32_presence_exact": fp32_parity["presence_matches"] == fp32_parity["videos"], | |
| "fp32_intervals_exact": fp32_parity["exact_interval_matches"] | |
| == fp32_parity["videos"], | |
| "int8_span_f1_within_limit": baseline_f1 - int8_f1 <= maximum_f1_drop, | |
| "int8_video_recall_not_lower": int8_recall >= baseline_recall, | |
| "int8_false_positives_not_higher": int8_false_positives | |
| <= baseline_false_positives, | |
| "int8_presence_exact": int8_parity["presence_matches"] | |
| == int8_parity["videos"], | |
| } | |
| status = "validated" if all(gates.values()) else "failed" | |
| package_files = { | |
| "model": _artifact(bundled_model_path), | |
| "operator_config": _artifact(bundled_operator_config_path), | |
| "tokenizer": _artifact(android_directory / "tokenizer.json"), | |
| "tokenizer_config": _artifact(android_directory / "tokenizer_config.json"), | |
| "model_config": _artifact(android_directory / "config.json"), | |
| "tokenizer_goldens": _artifact(tokenizer_goldens_path), | |
| "feedback_schema": _artifact(bundled_feedback_schema_path), | |
| } | |
| manifest = { | |
| "schema_version": 1, | |
| "name": package_name, | |
| "status": status, | |
| "source": { | |
| "checkpoint_path": str(checkpoint_path), | |
| "checkpoint_sha256": checkpoint_sha256, | |
| "base_model": "jhu-clsp/ettin-encoder-17m", | |
| "base_model_license": "MIT", | |
| }, | |
| "runtime": { | |
| "format": "ORT", | |
| "execution_provider": "CPUExecutionProvider", | |
| "quantization": "embedding_int8_per_channel", | |
| "opset": int(configuration.get("opset", 18)), | |
| }, | |
| "inference": { | |
| "labels": ["O", "B-SPONSOR", "I-SPONSOR", "L-SPONSOR", "U-SPONSOR"], | |
| "confidence_threshold": confidence_threshold, | |
| "max_length": max_length, | |
| "overlap_tokens": overlap_tokens, | |
| "merge_gap_characters": merge_gap_characters, | |
| "merge_gap_ms": merge_gap_ms, | |
| }, | |
| "package": { | |
| "directory": str(android_directory), | |
| "bytes": sum(int(artifact["bytes"]) for artifact in package_files.values()), | |
| "files": package_files, | |
| }, | |
| "validation": { | |
| "benchmark_sha256": dataset_provenance["benchmark_sha256"], | |
| "baseline_report": _artifact(baseline_report_path), | |
| "fp32_parity": fp32_parity, | |
| "int8_parity": int8_parity, | |
| "baseline_span_f1_at_iou_0.5": baseline_f1, | |
| "int8_span_f1_at_iou_0.5": int8_f1, | |
| "maximum_allowed_span_f1_drop": maximum_f1_drop, | |
| "gates": gates, | |
| "fp32_evaluation": evaluation_reports["fp32"], | |
| "int8_evaluation": evaluation_reports["int8"], | |
| }, | |
| "toolchain": { | |
| package: importlib.metadata.version(package) | |
| for package in ( | |
| "torch", | |
| "transformers", | |
| "onnx", | |
| "onnxruntime", | |
| "onnxscript", | |
| ) | |
| }, | |
| } | |
| write_json_atomic(android_directory / "manifest.json", manifest) | |
| write_json_atomic(report_path, manifest) | |
| return manifest | |