Download code/src/sponsor_detection/data/smart_segment_benchmark.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 15.6 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/data/smart_segment_benchmark.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/src/sponsor_detection/data/smart_segment_benchmark.py
-
curl -L -o smart_segment_benchmark.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/data/smart_segment_benchmark.py
15.6 kB
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import time | |
| from collections import Counter | |
| from dataclasses import dataclass | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Callable, Sequence | |
| from sponsor_detection.data.profile import sha256_file, write_json_atomic | |
| from sponsor_detection.data.scriptsmith_dataset import ( | |
| _is_english, | |
| _iter_subtitle_rows, | |
| _load_metadata, | |
| _parse_cues, | |
| reconstruct_caption_lines, | |
| ) | |
| from sponsor_detection.data.smart_segment_annotations import ( | |
| load_canonical_annotations, | |
| ) | |
| from sponsor_detection.inference.windowing import TranscriptCue, assemble_transcript | |
| BENCHMARK_VERSION = "smart-segment-benchmark/1" | |
| CONTENT_NEGATIVE_STRATUM = "content_negative" | |
| REVIEW_STATUS_PENDING = "pending" | |
| REVIEW_STATUS_REVIEWED = "reviewed" | |
| class BenchmarkCandidate: | |
| video_id: str | |
| channel_id: str | |
| stratum: str | |
| duration_ms: int | |
| language: str | |
| def _seed_order(candidate: BenchmarkCandidate, seed: str) -> str: | |
| return hashlib.sha256(f"{seed}\0{candidate.video_id}".encode("utf-8")).hexdigest() | |
| def select_candidates( | |
| candidates: Sequence[BenchmarkCandidate], | |
| *, | |
| targets: dict[str, int], | |
| seed: str, | |
| excluded_channels: set[str], | |
| ) -> list[BenchmarkCandidate]: | |
| """Pick channel-disjoint candidates, deterministically, towards per-stratum targets. | |
| At most one video per channel is selected, and channels already used by the | |
| frozen training splits are skipped, so the benchmark cannot leak into the | |
| data a candidate model was trained on. | |
| """ | |
| selected: list[BenchmarkCandidate] = [] | |
| counts: Counter[str] = Counter() | |
| used_channels: set[str] = set() | |
| for candidate in sorted( | |
| candidates, key=lambda item: _seed_order(item, seed) | |
| ): | |
| if candidate.channel_id in excluded_channels: | |
| continue | |
| if candidate.channel_id in used_channels: | |
| continue | |
| target = targets.get(candidate.stratum, 0) | |
| if counts[candidate.stratum] >= target: | |
| continue | |
| selected.append(candidate) | |
| counts[candidate.stratum] += 1 | |
| used_channels.add(candidate.channel_id) | |
| return selected | |
| def _stratum_for(present: set[str], categories: Sequence[str]) -> str: | |
| if present: | |
| return min(present, key=lambda category: categories.index(category)) | |
| return CONTENT_NEGATIVE_STRATUM | |
| def _load_excluded_channels(split_directories: Sequence[Path]) -> set[str]: | |
| import pyarrow.parquet as parquet | |
| channels: set[str] = set() | |
| for directory in split_directories: | |
| if not directory.is_dir(): | |
| continue | |
| for split in ("train", "validation", "test"): | |
| path = directory / f"{split}.parquet" | |
| if not path.is_file(): | |
| continue | |
| file = parquet.ParquetFile(path) | |
| for batch in file.iter_batches(columns=["channel_id"], batch_size=131_072): | |
| channels.update( | |
| str(value) | |
| for value in batch.column("channel_id").to_pylist() | |
| if value | |
| ) | |
| return channels | |
| def _transcript_payload(transcript) -> dict[str, object]: | |
| cues = [ | |
| { | |
| "start_ms": cue_range.start_ms, | |
| "end_ms": cue_range.end_ms, | |
| "text": transcript.text[cue_range.start_char : cue_range.end_char], | |
| } | |
| for cue_range in transcript.cue_ranges | |
| ] | |
| return { | |
| "text": transcript.text, | |
| "cues": cues, | |
| "sha256": hashlib.sha256(transcript.text.encode("utf-8")).hexdigest(), | |
| } | |
| def build_benchmark_candidates( | |
| subtitles_directory: Path, | |
| metadata_directory: Path, | |
| annotations_path: Path, | |
| exclude_split_directories: Sequence[Path], | |
| output_path: Path, | |
| manifest_path: Path, | |
| *, | |
| categories: Sequence[str], | |
| targets: dict[str, int], | |
| seed: str, | |
| language_prefixes: Sequence[str] = ("en",), | |
| maximum_video_duration_seconds: int = 14400, | |
| minimum_cues: int = 25, | |
| reserved_directory: Path | None = None, | |
| progress_callback: Callable[[str], None] | None = None, | |
| ) -> dict[str, object]: | |
| """Build a full-transcript review queue for the V1 categories. | |
| A reviewer sees the complete transcript rather than only the intervals the | |
| mirror already knows about, so content-heavy negatives and missed spans are | |
| discoverable. | |
| When ``reserved_directory`` is set, the selected videos and channels are | |
| written as a split-shaped exclusion the training builder consumes. The | |
| benchmark must be reserved **before** the training splits are built; | |
| otherwise the two draw from the same finite channel pool and no disjoint | |
| benchmark can exist. | |
| """ | |
| started = time.monotonic() | |
| if progress_callback: | |
| progress_callback("loading metadata and annotations") | |
| metadata = _load_metadata(metadata_directory) | |
| annotations = load_canonical_annotations( | |
| annotations_path, categories=list(categories), eligible_only=True | |
| ) | |
| excluded_channels = _load_excluded_channels(exclude_split_directories) | |
| if progress_callback: | |
| progress_callback("scanning transcripts") | |
| candidates: list[BenchmarkCandidate] = [] | |
| for row in _iter_subtitle_rows(subtitles_directory): | |
| video_id = str(row["video_id"]) | |
| if video_id not in metadata: | |
| continue | |
| if not _is_english(row["language"], language_prefixes): | |
| continue | |
| meta = metadata[video_id] | |
| if meta.get("availability") not in (None, "public"): | |
| continue | |
| if meta.get("live_status") == "was_live": | |
| continue | |
| duration = meta.get("duration") | |
| if not isinstance(duration, int) or duration > maximum_video_duration_seconds: | |
| continue | |
| channel_id = meta.get("channel_id") | |
| if not channel_id: | |
| continue | |
| if len(_parse_cues(str(row["segments_json"]))) < minimum_cues: | |
| continue | |
| present = {record.category for record in annotations.get(video_id, [])} | |
| candidates.append( | |
| BenchmarkCandidate( | |
| video_id=video_id, | |
| channel_id=str(channel_id), | |
| stratum=_stratum_for(present, categories), | |
| duration_ms=duration * 1000, | |
| language=str(row["language"]), | |
| ) | |
| ) | |
| selected = select_candidates( | |
| candidates, targets=targets, seed=seed, excluded_channels=excluded_channels | |
| ) | |
| selected_ids = {candidate.video_id for candidate in selected} | |
| # Materialize transcripts for the selected videos only, so the review queue | |
| # does not hold the whole corpus in memory. | |
| if progress_callback: | |
| progress_callback("materializing selected transcripts") | |
| transcripts: dict[str, dict[str, object]] = {} | |
| for row in _iter_subtitle_rows(subtitles_directory): | |
| video_id = str(row["video_id"]) | |
| if video_id not in selected_ids: | |
| continue | |
| reconstructed = reconstruct_caption_lines( | |
| _parse_cues(str(row["segments_json"])) | |
| ) | |
| transcript = assemble_transcript( | |
| [ | |
| TranscriptCue(index=index, start_ms=start, end_ms=end, text=text) | |
| for index, (start, end, text) in enumerate(reconstructed) | |
| ] | |
| ) | |
| if not transcript.text.strip(): | |
| continue | |
| transcripts[video_id] = _transcript_payload(transcript) | |
| selected = [ | |
| candidate for candidate in selected if candidate.video_id in transcripts | |
| ] | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| with output_path.open("w", encoding="utf-8") as sink: | |
| for candidate in selected: | |
| sink.write( | |
| json.dumps( | |
| { | |
| "video_id": candidate.video_id, | |
| "channel_id": candidate.channel_id, | |
| "stratum": candidate.stratum, | |
| "duration_ms": candidate.duration_ms, | |
| "language": candidate.language, | |
| "categories": list(categories), | |
| "transcript": transcripts[candidate.video_id], | |
| "annotation": { | |
| "status": REVIEW_STATUS_PENDING, | |
| "reviewer": None, | |
| "content_confirmed": False, | |
| "intervals": {category: [] for category in categories}, | |
| }, | |
| }, | |
| sort_keys=True, | |
| ) | |
| + "\n" | |
| ) | |
| counts = Counter(candidate.stratum for candidate in selected) | |
| reserved: dict[str, object] = {"path": None, "sha256": None, "videos": 0} | |
| if reserved_directory is not None: | |
| import pyarrow as pa | |
| import pyarrow.parquet as parquet | |
| reserved_directory.mkdir(parents=True, exist_ok=True) | |
| reserved_path = reserved_directory / "train.parquet" | |
| parquet.write_table( | |
| pa.table( | |
| { | |
| "video_id": [candidate.video_id for candidate in selected], | |
| "channel_id": [candidate.channel_id for candidate in selected], | |
| } | |
| ), | |
| reserved_path, | |
| compression="zstd", | |
| ) | |
| reserved = { | |
| "path": str(reserved_path), | |
| "sha256": sha256_file(reserved_path), | |
| "videos": len(selected), | |
| } | |
| manifest = { | |
| "version": BENCHMARK_VERSION, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "status": "annotation_pending", | |
| "configuration": { | |
| "categories": list(categories), | |
| "targets": targets, | |
| "seed": seed, | |
| "language_prefixes": list(language_prefixes), | |
| "maximum_video_duration_seconds": maximum_video_duration_seconds, | |
| "minimum_cues": minimum_cues, | |
| }, | |
| "sources": { | |
| "subtitles_directory": str(subtitles_directory), | |
| "metadata_directory": str(metadata_directory), | |
| "annotations_path": str(annotations_path), | |
| "annotations_sha256": sha256_file(annotations_path), | |
| }, | |
| "counts": { | |
| "candidates": len(candidates), | |
| "selected": len(selected), | |
| "excluded_channels": len(excluded_channels), | |
| "by_stratum": dict(sorted(counts.items())), | |
| }, | |
| "output": { | |
| "path": str(output_path), | |
| "bytes": output_path.stat().st_size, | |
| "sha256": sha256_file(output_path), | |
| }, | |
| "reserved": reserved, | |
| "elapsed_seconds": round(time.monotonic() - started, 3), | |
| } | |
| write_json_atomic(manifest_path, manifest) | |
| return manifest | |
| def _validated_intervals( | |
| record: dict[str, object], category: str | |
| ) -> list[dict[str, int]]: | |
| annotation = record.get("annotation") | |
| if not isinstance(annotation, dict): | |
| raise ValueError(f"video {record.get('video_id')} has no annotation block") | |
| intervals = annotation.get("intervals") | |
| if not isinstance(intervals, dict): | |
| raise ValueError(f"video {record.get('video_id')} has no interval map") | |
| category_intervals = intervals.get(category, []) | |
| if not isinstance(category_intervals, list): | |
| raise ValueError( | |
| f"video {record.get('video_id')} category {category} intervals must be a list" | |
| ) | |
| validated: list[dict[str, int]] = [] | |
| for interval in category_intervals: | |
| start = int(interval["start_ms"]) | |
| end = int(interval["end_ms"]) | |
| if start < 0 or end <= start: | |
| raise ValueError( | |
| f"video {record.get('video_id')} has an invalid {category} interval" | |
| ) | |
| validated.append({"start_ms": start, "end_ms": end}) | |
| return validated | |
| def freeze_benchmark( | |
| review_path: Path, | |
| output_path: Path, | |
| manifest_path: Path, | |
| *, | |
| categories: Sequence[str], | |
| ) -> dict[str, object]: | |
| """Freeze a reviewed benchmark and derive confirmed negatives from it. | |
| A fully reviewed video that contains no interval for any configured | |
| category becomes a confirmed negative for every one of them, with the | |
| reviewer recorded as the evidence source. | |
| """ | |
| records: list[dict[str, object]] = [] | |
| with review_path.open("r", encoding="utf-8") as source: | |
| for line_number, line in enumerate(source, start=1): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| record = json.loads(line) | |
| annotation = record.get("annotation") | |
| if not isinstance(annotation, dict) or annotation.get("status") != REVIEW_STATUS_REVIEWED: | |
| raise ValueError( | |
| f"review line {line_number} is not marked {REVIEW_STATUS_REVIEWED}" | |
| ) | |
| reviewer = annotation.get("reviewer") | |
| if not reviewer: | |
| raise ValueError(f"review line {line_number} has no reviewer") | |
| records.append(record) | |
| frozen: list[dict[str, object]] = [] | |
| confirmed_negatives = 0 | |
| positive_by_category: Counter[str] = Counter() | |
| for record in records: | |
| categories_intervals = { | |
| category: _validated_intervals(record, category) | |
| for category in categories | |
| } | |
| has_any = any(categories_intervals.values()) | |
| content_confirmed = bool(record["annotation"].get("content_confirmed")) | |
| if not has_any and not content_confirmed: | |
| raise ValueError( | |
| f"video {record['video_id']} has no intervals and is not confirmed content" | |
| ) | |
| for category, intervals in categories_intervals.items(): | |
| if intervals: | |
| positive_by_category[category] += 1 | |
| if not has_any: | |
| confirmed_negatives += 1 | |
| frozen.append( | |
| { | |
| "video_id": record["video_id"], | |
| "channel_id": record["channel_id"], | |
| "stratum": record["stratum"], | |
| "reviewer": str(record["annotation"]["reviewer"]), | |
| "categories": categories_intervals, | |
| "confirmed_negative": not has_any, | |
| "transcript_sha256": record["transcript"]["sha256"], | |
| } | |
| ) | |
| transcript_digest = hashlib.sha256() | |
| for record in sorted(frozen, key=lambda item: str(item["video_id"])): | |
| transcript_digest.update( | |
| f"{record['video_id']}\0{record['transcript_sha256']}\n".encode("utf-8") | |
| ) | |
| with output_path.open("w", encoding="utf-8") as sink: | |
| for record in sorted(frozen, key=lambda item: str(item["video_id"])): | |
| sink.write(json.dumps(record, sort_keys=True) + "\n") | |
| manifest = { | |
| "version": BENCHMARK_VERSION, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "status": "frozen", | |
| "categories": list(categories), | |
| "source_review_sha256": sha256_file(review_path), | |
| "transcript_set_sha256": transcript_digest.hexdigest(), | |
| "counts": { | |
| "videos": len(frozen), | |
| "confirmed_negative_videos": confirmed_negatives, | |
| "positive_videos_by_category": dict(sorted(positive_by_category.items())), | |
| }, | |
| "output": { | |
| "path": str(output_path), | |
| "bytes": output_path.stat().st_size, | |
| "sha256": sha256_file(output_path), | |
| }, | |
| } | |
| write_json_atomic(manifest_path, manifest) | |
| return manifest | |