Download code/src/sponsor_detection/data/benchmark.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 14.3 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/data/benchmark.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/src/sponsor_detection/data/benchmark.py
-
curl -L -o benchmark.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/data/benchmark.py
14.3 kB
| from __future__ import annotations | |
| import csv | |
| import hashlib | |
| import heapq | |
| import json | |
| import os | |
| import tempfile | |
| import time | |
| from collections import Counter | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Callable, Iterable | |
| from sponsor_detection.data.profile import sha256_file, write_json_atomic | |
| HARD_NEGATIVE_CATEGORIES = frozenset( | |
| {"exclusive_access", "interaction", "selfpromo"} | |
| ) | |
| ORDINARY_NEGATIVE_CATEGORIES = frozenset( | |
| {"filler", "hook", "intro", "music_offtopic", "outro", "poi_highlight", "preview"} | |
| ) | |
| TARGET_CLASSES = ("sponsor_positive", "hard_negative", "ordinary_negative") | |
| def _score(seed: str, namespace: str, video_id: str) -> int: | |
| digest = hashlib.blake2b( | |
| f"{seed}\0{namespace}\0{video_id}".encode("utf-8"), digest_size=8 | |
| ).digest() | |
| return int.from_bytes(digest, "big") | |
| class _BottomK: | |
| def __init__(self, capacity: int, seed: str, namespace: str) -> None: | |
| self.capacity = capacity | |
| self.seed = seed | |
| self.namespace = namespace | |
| self.heap: list[tuple[int, str]] = [] | |
| self.records: dict[str, dict[str, object]] = {} | |
| def add(self, video_id: str, record: dict[str, object]) -> None: | |
| score = _score(self.seed, self.namespace, video_id) | |
| if video_id in self.records: | |
| self.records[video_id] = record | |
| return | |
| if len(self.heap) < self.capacity: | |
| heapq.heappush(self.heap, (-score, video_id)) | |
| elif score < -self.heap[0][0]: | |
| _, evicted = heapq.heapreplace(self.heap, (-score, video_id)) | |
| del self.records[evicted] | |
| else: | |
| return | |
| self.records[video_id] = record | |
| def sorted_records(self) -> list[dict[str, object]]: | |
| return sorted( | |
| self.records.values(), | |
| key=lambda record: _score( | |
| self.seed, self.namespace, str(record["video_id"]) | |
| ), | |
| ) | |
| def write_json_lines_atomic(path: Path, records: Iterable[dict[str, object]]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| descriptor, temporary_name = tempfile.mkstemp( | |
| dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", text=True | |
| ) | |
| try: | |
| with os.fdopen(descriptor, "w", encoding="utf-8") as output: | |
| for record in records: | |
| json.dump(record, output, sort_keys=True, separators=(",", ":")) | |
| output.write("\n") | |
| os.replace(temporary_name, path) | |
| except BaseException: | |
| Path(temporary_name).unlink(missing_ok=True) | |
| raise | |
| def _jsonl_video_ids(path: Path | None) -> set[str]: | |
| if path is None or not path.is_file(): | |
| return set() | |
| video_ids: set[str] = set() | |
| with path.open("r", encoding="utf-8") as source: | |
| for line_number, line in enumerate(source, start=1): | |
| try: | |
| video_ids.add(str(json.loads(line)["video_id"])) | |
| except (KeyError, TypeError, json.JSONDecodeError) as error: | |
| raise ValueError(f"invalid JSONL record at {path}:{line_number}") from error | |
| return video_ids | |
| def _parquet_video_ids(directory: Path) -> set[str]: | |
| try: | |
| import pyarrow.parquet as pq | |
| except ImportError as error: | |
| raise RuntimeError( | |
| "Benchmark sampling requires the data extra: pip install -e '.[data]'" | |
| ) from error | |
| video_ids: set[str] = set() | |
| paths = sorted(directory.glob("*.parquet")) | |
| if not paths: | |
| raise ValueError(f"training directory contains no Parquet splits: {directory}") | |
| for path in paths: | |
| parquet = pq.ParquetFile(path) | |
| for batch in parquet.iter_batches(columns=["video_id"], batch_size=131_072): | |
| video_ids.update(str(value) for value in batch.column("video_id").to_pylist()) | |
| return video_ids | |
| def _eligible_mirror_row(row: dict[str, str]) -> bool: | |
| try: | |
| return ( | |
| row["service"] == "YouTube" | |
| and int(row["hidden"]) == 0 | |
| and int(row["shadowHidden"]) == 0 | |
| and int(row["votes"]) >= 0 | |
| and float(row["startTime"]) >= 0 | |
| and float(row["endTime"]) > float(row["startTime"]) | |
| ) | |
| except (KeyError, TypeError, ValueError): | |
| return False | |
| def _positive_candidates( | |
| labels_path: Path, | |
| *, | |
| excluded_video_ids: set[str], | |
| capacity: int, | |
| seed: str, | |
| batch_size: int, | |
| ) -> tuple[list[dict[str, object]], int]: | |
| import pyarrow.parquet as pq | |
| sampler = _BottomK(capacity, seed, "sponsor_positive") | |
| scanned_rows = 0 | |
| columns = [ | |
| "video_id", | |
| "segment_id", | |
| "start_ms", | |
| "end_ms", | |
| "votes", | |
| "locked", | |
| "is_eligible", | |
| ] | |
| parquet = pq.ParquetFile(labels_path) | |
| for batch in parquet.iter_batches(columns=["video_id", "is_eligible"], batch_size=batch_size): | |
| values = batch.to_pydict() | |
| for index, video_id_value in enumerate(values["video_id"]): | |
| scanned_rows += 1 | |
| video_id = str(video_id_value) | |
| if not values["is_eligible"][index] or video_id in excluded_video_ids: | |
| continue | |
| sampler.add( | |
| video_id, | |
| { | |
| "video_id": video_id, | |
| "benchmark_class": "sponsor_positive", | |
| "source_categories": ["sponsor"], | |
| "segments": [], | |
| }, | |
| ) | |
| selected = set(sampler.records) | |
| for batch in parquet.iter_batches(columns=columns, batch_size=batch_size): | |
| values = batch.to_pydict() | |
| for index, video_id_value in enumerate(values["video_id"]): | |
| video_id = str(video_id_value) | |
| if video_id not in selected or not values["is_eligible"][index]: | |
| continue | |
| sampler.records[video_id]["segments"].append( | |
| { | |
| "segment_id": str(values["segment_id"][index]), | |
| "start_ms": int(values["start_ms"][index]), | |
| "end_ms": int(values["end_ms"][index]), | |
| "votes": int(values["votes"][index]), | |
| "locked": bool(values["locked"][index]), | |
| "category": "sponsor", | |
| } | |
| ) | |
| for record in sampler.records.values(): | |
| record["segments"].sort( | |
| key=lambda segment: (segment["start_ms"], segment["end_ms"]) | |
| ) | |
| return sampler.sorted_records(), scanned_rows | |
| def _negative_candidates( | |
| mirror_csv_path: Path, | |
| *, | |
| excluded_video_ids: set[str], | |
| pool_per_class: int, | |
| seed: str, | |
| sample_modulus: int, | |
| progress_every_rows: int, | |
| progress_callback: Callable[[int, float], None] | None, | |
| ) -> tuple[dict[str, list[dict[str, object]]], dict[str, int]]: | |
| sampled: dict[str, dict[str, object]] = {} | |
| scanned_rows = eligible_rows = 0 | |
| started = time.monotonic() | |
| next_progress = progress_every_rows | |
| with mirror_csv_path.open("r", encoding="utf-8", newline="") as source: | |
| reader = csv.DictReader(source) | |
| for row in reader: | |
| scanned_rows += 1 | |
| video_id = row.get("videoID", "") | |
| if ( | |
| video_id in excluded_video_ids | |
| or _score(seed, "negative_universe", video_id) % sample_modulus | |
| ): | |
| continue | |
| if not _eligible_mirror_row(row): | |
| continue | |
| eligible_rows += 1 | |
| record = sampled.setdefault( | |
| video_id, | |
| {"video_id": video_id, "categories": set(), "segments": []}, | |
| ) | |
| category = row["category"] | |
| record["categories"].add(category) | |
| if category in HARD_NEGATIVE_CATEGORIES | ORDINARY_NEGATIVE_CATEGORIES: | |
| record["segments"].append( | |
| { | |
| "segment_id": row["UUID"], | |
| "start_ms": round(float(row["startTime"]) * 1000), | |
| "end_ms": round(float(row["endTime"]) * 1000), | |
| "votes": int(row["votes"]), | |
| "locked": int(row["locked"]) != 0, | |
| "category": category, | |
| } | |
| ) | |
| if progress_callback and progress_every_rows and scanned_rows >= next_progress: | |
| progress_callback(scanned_rows, time.monotonic() - started) | |
| next_progress += progress_every_rows | |
| samplers = { | |
| name: _BottomK(pool_per_class, seed, name) | |
| for name in ("hard_negative", "ordinary_negative") | |
| } | |
| classifications: Counter[str] = Counter() | |
| for video_id, raw_record in sampled.items(): | |
| categories = set(raw_record["categories"]) | |
| if "sponsor" in categories: | |
| classifications["excluded_sponsor"] += 1 | |
| continue | |
| if categories & HARD_NEGATIVE_CATEGORIES: | |
| benchmark_class = "hard_negative" | |
| elif categories & ORDINARY_NEGATIVE_CATEGORIES: | |
| benchmark_class = "ordinary_negative" | |
| else: | |
| classifications["no_target_category"] += 1 | |
| continue | |
| classifications[benchmark_class] += 1 | |
| samplers[benchmark_class].add( | |
| video_id, | |
| { | |
| "video_id": video_id, | |
| "benchmark_class": benchmark_class, | |
| "source_categories": sorted(categories), | |
| "segments": sorted( | |
| raw_record["segments"], | |
| key=lambda segment: (segment["start_ms"], segment["end_ms"]), | |
| ), | |
| }, | |
| ) | |
| records = {name: sampler.sorted_records() for name, sampler in samplers.items()} | |
| counts = { | |
| "scanned_mirror_rows": scanned_rows, | |
| "sampled_eligible_rows": eligible_rows, | |
| "sampled_videos": len(sampled), | |
| **dict(classifications), | |
| } | |
| return records, counts | |
| def build_mixed_benchmark_candidates( | |
| mirror_csv_path: Path, | |
| labels_path: Path, | |
| labels_manifest_path: Path, | |
| training_directory: Path, | |
| output_path: Path, | |
| manifest_path: Path, | |
| *, | |
| existing_pilot_path: Path | None, | |
| target_per_class: int, | |
| pool_per_class: int, | |
| seed: str, | |
| negative_sample_modulus: int = 64, | |
| batch_size: int = 131_072, | |
| progress_every_rows: int = 1_000_000, | |
| progress_callback: Callable[[int, float], None] | None = None, | |
| ) -> dict[str, object]: | |
| if target_per_class <= 0: | |
| raise ValueError("target per class must be positive") | |
| if pool_per_class < target_per_class: | |
| raise ValueError("pool per class cannot be smaller than the target") | |
| if negative_sample_modulus <= 0: | |
| raise ValueError("negative sample modulus must be positive") | |
| labels_manifest = json.loads(labels_manifest_path.read_text(encoding="utf-8")) | |
| labels_sha256 = labels_manifest.get("output", {}).get("sha256") | |
| if not labels_sha256 or sha256_file(labels_path) != labels_sha256: | |
| raise ValueError("sponsor labels do not match their manifest") | |
| started = time.monotonic() | |
| training_video_ids = _parquet_video_ids(training_directory) | |
| prior_pilot_video_ids = _jsonl_video_ids(existing_pilot_path) | |
| excluded_video_ids = training_video_ids | prior_pilot_video_ids | |
| positives, scanned_label_rows = _positive_candidates( | |
| labels_path, | |
| excluded_video_ids=excluded_video_ids, | |
| capacity=pool_per_class, | |
| seed=seed, | |
| batch_size=batch_size, | |
| ) | |
| negatives, mirror_counts = _negative_candidates( | |
| mirror_csv_path, | |
| excluded_video_ids=excluded_video_ids, | |
| pool_per_class=pool_per_class, | |
| seed=seed, | |
| sample_modulus=negative_sample_modulus, | |
| progress_every_rows=progress_every_rows, | |
| progress_callback=progress_callback, | |
| ) | |
| records_by_class = { | |
| "sponsor_positive": positives, | |
| "hard_negative": negatives["hard_negative"], | |
| "ordinary_negative": negatives["ordinary_negative"], | |
| } | |
| short_classes = { | |
| name: len(records) | |
| for name, records in records_by_class.items() | |
| if len(records) < pool_per_class | |
| } | |
| if short_classes: | |
| raise ValueError(f"insufficient benchmark candidates: {short_classes}") | |
| interleaved: list[dict[str, object]] = [] | |
| for rank in range(pool_per_class): | |
| for benchmark_class in TARGET_CLASSES: | |
| record = dict(records_by_class[benchmark_class][rank]) | |
| record["selection_rank"] = rank | |
| record["sample_hash"] = ( | |
| f"{_score(seed, benchmark_class, str(record['video_id'])):016x}" | |
| ) | |
| interleaved.append(record) | |
| write_json_lines_atomic(output_path, interleaved) | |
| report = { | |
| "schema_version": 1, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "status": "acquisition_pending", | |
| "sources": { | |
| "mirror_path": str(mirror_csv_path), | |
| "mirror_bytes": mirror_csv_path.stat().st_size, | |
| "labels_path": str(labels_path), | |
| "labels_sha256": labels_sha256, | |
| "training_directory": str(training_directory), | |
| }, | |
| "sampling": { | |
| "seed": seed, | |
| "method": "stable hash bucket followed by per-class bottom-k", | |
| "target_per_class": target_per_class, | |
| "pool_per_class": pool_per_class, | |
| "negative_sample_modulus": negative_sample_modulus, | |
| "queue_order": "round-robin by class and within-class stable rank", | |
| }, | |
| "exclusions": { | |
| "training_videos": len(training_video_ids), | |
| "prior_pilot_videos": len(prior_pilot_video_ids), | |
| "unique_excluded_videos": len(excluded_video_ids), | |
| }, | |
| "counts": { | |
| "scanned_label_rows": scanned_label_rows, | |
| **mirror_counts, | |
| "queue_videos": len(interleaved), | |
| "queue_by_class": { | |
| name: len(records) for name, records in records_by_class.items() | |
| }, | |
| }, | |
| "output": { | |
| "path": str(output_path), | |
| "bytes": output_path.stat().st_size, | |
| "sha256": sha256_file(output_path), | |
| }, | |
| "elapsed_seconds": round(time.monotonic() - started, 3), | |
| } | |
| write_json_atomic(manifest_path, report) | |
| return report | |