flow-smart-segments-20261004 / code /src /sponsor_detection /data /smart_segment_benchmark.py
CuriousDragon's picture
Publish verified Flow Smart Segments curated-category-encoder-ensemble-20261004
8fcf7c9 verified
Raw History Blame Contribute Delete
15.6 kB
from __future__ import annotations
import hashlib
import json
import time
from collections import Counter
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Callable, Sequence
from sponsor_detection.data.profile import sha256_file, write_json_atomic
from sponsor_detection.data.scriptsmith_dataset import (
_is_english,
_iter_subtitle_rows,
_load_metadata,
_parse_cues,
reconstruct_caption_lines,
)
from sponsor_detection.data.smart_segment_annotations import (
load_canonical_annotations,
)
from sponsor_detection.inference.windowing import TranscriptCue, assemble_transcript
BENCHMARK_VERSION = "smart-segment-benchmark/1"
CONTENT_NEGATIVE_STRATUM = "content_negative"
REVIEW_STATUS_PENDING = "pending"
REVIEW_STATUS_REVIEWED = "reviewed"
@dataclass(frozen=True, slots=True)
class BenchmarkCandidate:
video_id: str
channel_id: str
stratum: str
duration_ms: int
language: str
def _seed_order(candidate: BenchmarkCandidate, seed: str) -> str:
return hashlib.sha256(f"{seed}\0{candidate.video_id}".encode("utf-8")).hexdigest()
def select_candidates(
candidates: Sequence[BenchmarkCandidate],
*,
targets: dict[str, int],
seed: str,
excluded_channels: set[str],
) -> list[BenchmarkCandidate]:
"""Pick channel-disjoint candidates, deterministically, towards per-stratum targets.
At most one video per channel is selected, and channels already used by the
frozen training splits are skipped, so the benchmark cannot leak into the
data a candidate model was trained on.
"""
selected: list[BenchmarkCandidate] = []
counts: Counter[str] = Counter()
used_channels: set[str] = set()
for candidate in sorted(
candidates, key=lambda item: _seed_order(item, seed)
):
if candidate.channel_id in excluded_channels:
continue
if candidate.channel_id in used_channels:
continue
target = targets.get(candidate.stratum, 0)
if counts[candidate.stratum] >= target:
continue
selected.append(candidate)
counts[candidate.stratum] += 1
used_channels.add(candidate.channel_id)
return selected
def _stratum_for(present: set[str], categories: Sequence[str]) -> str:
if present:
return min(present, key=lambda category: categories.index(category))
return CONTENT_NEGATIVE_STRATUM
def _load_excluded_channels(split_directories: Sequence[Path]) -> set[str]:
import pyarrow.parquet as parquet
channels: set[str] = set()
for directory in split_directories:
if not directory.is_dir():
continue
for split in ("train", "validation", "test"):
path = directory / f"{split}.parquet"
if not path.is_file():
continue
file = parquet.ParquetFile(path)
for batch in file.iter_batches(columns=["channel_id"], batch_size=131_072):
channels.update(
str(value)
for value in batch.column("channel_id").to_pylist()
if value
)
return channels
def _transcript_payload(transcript) -> dict[str, object]:
cues = [
{
"start_ms": cue_range.start_ms,
"end_ms": cue_range.end_ms,
"text": transcript.text[cue_range.start_char : cue_range.end_char],
}
for cue_range in transcript.cue_ranges
]
return {
"text": transcript.text,
"cues": cues,
"sha256": hashlib.sha256(transcript.text.encode("utf-8")).hexdigest(),
}
def build_benchmark_candidates(
subtitles_directory: Path,
metadata_directory: Path,
annotations_path: Path,
exclude_split_directories: Sequence[Path],
output_path: Path,
manifest_path: Path,
*,
categories: Sequence[str],
targets: dict[str, int],
seed: str,
language_prefixes: Sequence[str] = ("en",),
maximum_video_duration_seconds: int = 14400,
minimum_cues: int = 25,
reserved_directory: Path | None = None,
progress_callback: Callable[[str], None] | None = None,
) -> dict[str, object]:
"""Build a full-transcript review queue for the V1 categories.
A reviewer sees the complete transcript rather than only the intervals the
mirror already knows about, so content-heavy negatives and missed spans are
discoverable.
When ``reserved_directory`` is set, the selected videos and channels are
written as a split-shaped exclusion the training builder consumes. The
benchmark must be reserved **before** the training splits are built;
otherwise the two draw from the same finite channel pool and no disjoint
benchmark can exist.
"""
started = time.monotonic()
if progress_callback:
progress_callback("loading metadata and annotations")
metadata = _load_metadata(metadata_directory)
annotations = load_canonical_annotations(
annotations_path, categories=list(categories), eligible_only=True
)
excluded_channels = _load_excluded_channels(exclude_split_directories)
if progress_callback:
progress_callback("scanning transcripts")
candidates: list[BenchmarkCandidate] = []
for row in _iter_subtitle_rows(subtitles_directory):
video_id = str(row["video_id"])
if video_id not in metadata:
continue
if not _is_english(row["language"], language_prefixes):
continue
meta = metadata[video_id]
if meta.get("availability") not in (None, "public"):
continue
if meta.get("live_status") == "was_live":
continue
duration = meta.get("duration")
if not isinstance(duration, int) or duration > maximum_video_duration_seconds:
continue
channel_id = meta.get("channel_id")
if not channel_id:
continue
if len(_parse_cues(str(row["segments_json"]))) < minimum_cues:
continue
present = {record.category for record in annotations.get(video_id, [])}
candidates.append(
BenchmarkCandidate(
video_id=video_id,
channel_id=str(channel_id),
stratum=_stratum_for(present, categories),
duration_ms=duration * 1000,
language=str(row["language"]),
)
)
selected = select_candidates(
candidates, targets=targets, seed=seed, excluded_channels=excluded_channels
)
selected_ids = {candidate.video_id for candidate in selected}
# Materialize transcripts for the selected videos only, so the review queue
# does not hold the whole corpus in memory.
if progress_callback:
progress_callback("materializing selected transcripts")
transcripts: dict[str, dict[str, object]] = {}
for row in _iter_subtitle_rows(subtitles_directory):
video_id = str(row["video_id"])
if video_id not in selected_ids:
continue
reconstructed = reconstruct_caption_lines(
_parse_cues(str(row["segments_json"]))
)
transcript = assemble_transcript(
[
TranscriptCue(index=index, start_ms=start, end_ms=end, text=text)
for index, (start, end, text) in enumerate(reconstructed)
]
)
if not transcript.text.strip():
continue
transcripts[video_id] = _transcript_payload(transcript)
selected = [
candidate for candidate in selected if candidate.video_id in transcripts
]
output_path.parent.mkdir(parents=True, exist_ok=True)
with output_path.open("w", encoding="utf-8") as sink:
for candidate in selected:
sink.write(
json.dumps(
{
"video_id": candidate.video_id,
"channel_id": candidate.channel_id,
"stratum": candidate.stratum,
"duration_ms": candidate.duration_ms,
"language": candidate.language,
"categories": list(categories),
"transcript": transcripts[candidate.video_id],
"annotation": {
"status": REVIEW_STATUS_PENDING,
"reviewer": None,
"content_confirmed": False,
"intervals": {category: [] for category in categories},
},
},
sort_keys=True,
)
+ "\n"
)
counts = Counter(candidate.stratum for candidate in selected)
reserved: dict[str, object] = {"path": None, "sha256": None, "videos": 0}
if reserved_directory is not None:
import pyarrow as pa
import pyarrow.parquet as parquet
reserved_directory.mkdir(parents=True, exist_ok=True)
reserved_path = reserved_directory / "train.parquet"
parquet.write_table(
pa.table(
{
"video_id": [candidate.video_id for candidate in selected],
"channel_id": [candidate.channel_id for candidate in selected],
}
),
reserved_path,
compression="zstd",
)
reserved = {
"path": str(reserved_path),
"sha256": sha256_file(reserved_path),
"videos": len(selected),
}
manifest = {
"version": BENCHMARK_VERSION,
"generated_at": datetime.now(tz=UTC).isoformat(),
"status": "annotation_pending",
"configuration": {
"categories": list(categories),
"targets": targets,
"seed": seed,
"language_prefixes": list(language_prefixes),
"maximum_video_duration_seconds": maximum_video_duration_seconds,
"minimum_cues": minimum_cues,
},
"sources": {
"subtitles_directory": str(subtitles_directory),
"metadata_directory": str(metadata_directory),
"annotations_path": str(annotations_path),
"annotations_sha256": sha256_file(annotations_path),
},
"counts": {
"candidates": len(candidates),
"selected": len(selected),
"excluded_channels": len(excluded_channels),
"by_stratum": dict(sorted(counts.items())),
},
"output": {
"path": str(output_path),
"bytes": output_path.stat().st_size,
"sha256": sha256_file(output_path),
},
"reserved": reserved,
"elapsed_seconds": round(time.monotonic() - started, 3),
}
write_json_atomic(manifest_path, manifest)
return manifest
def _validated_intervals(
record: dict[str, object], category: str
) -> list[dict[str, int]]:
annotation = record.get("annotation")
if not isinstance(annotation, dict):
raise ValueError(f"video {record.get('video_id')} has no annotation block")
intervals = annotation.get("intervals")
if not isinstance(intervals, dict):
raise ValueError(f"video {record.get('video_id')} has no interval map")
category_intervals = intervals.get(category, [])
if not isinstance(category_intervals, list):
raise ValueError(
f"video {record.get('video_id')} category {category} intervals must be a list"
)
validated: list[dict[str, int]] = []
for interval in category_intervals:
start = int(interval["start_ms"])
end = int(interval["end_ms"])
if start < 0 or end <= start:
raise ValueError(
f"video {record.get('video_id')} has an invalid {category} interval"
)
validated.append({"start_ms": start, "end_ms": end})
return validated
def freeze_benchmark(
review_path: Path,
output_path: Path,
manifest_path: Path,
*,
categories: Sequence[str],
) -> dict[str, object]:
"""Freeze a reviewed benchmark and derive confirmed negatives from it.
A fully reviewed video that contains no interval for any configured
category becomes a confirmed negative for every one of them, with the
reviewer recorded as the evidence source.
"""
records: list[dict[str, object]] = []
with review_path.open("r", encoding="utf-8") as source:
for line_number, line in enumerate(source, start=1):
line = line.strip()
if not line:
continue
record = json.loads(line)
annotation = record.get("annotation")
if not isinstance(annotation, dict) or annotation.get("status") != REVIEW_STATUS_REVIEWED:
raise ValueError(
f"review line {line_number} is not marked {REVIEW_STATUS_REVIEWED}"
)
reviewer = annotation.get("reviewer")
if not reviewer:
raise ValueError(f"review line {line_number} has no reviewer")
records.append(record)
frozen: list[dict[str, object]] = []
confirmed_negatives = 0
positive_by_category: Counter[str] = Counter()
for record in records:
categories_intervals = {
category: _validated_intervals(record, category)
for category in categories
}
has_any = any(categories_intervals.values())
content_confirmed = bool(record["annotation"].get("content_confirmed"))
if not has_any and not content_confirmed:
raise ValueError(
f"video {record['video_id']} has no intervals and is not confirmed content"
)
for category, intervals in categories_intervals.items():
if intervals:
positive_by_category[category] += 1
if not has_any:
confirmed_negatives += 1
frozen.append(
{
"video_id": record["video_id"],
"channel_id": record["channel_id"],
"stratum": record["stratum"],
"reviewer": str(record["annotation"]["reviewer"]),
"categories": categories_intervals,
"confirmed_negative": not has_any,
"transcript_sha256": record["transcript"]["sha256"],
}
)
transcript_digest = hashlib.sha256()
for record in sorted(frozen, key=lambda item: str(item["video_id"])):
transcript_digest.update(
f"{record['video_id']}\0{record['transcript_sha256']}\n".encode("utf-8")
)
with output_path.open("w", encoding="utf-8") as sink:
for record in sorted(frozen, key=lambda item: str(item["video_id"])):
sink.write(json.dumps(record, sort_keys=True) + "\n")
manifest = {
"version": BENCHMARK_VERSION,
"generated_at": datetime.now(tz=UTC).isoformat(),
"status": "frozen",
"categories": list(categories),
"source_review_sha256": sha256_file(review_path),
"transcript_set_sha256": transcript_digest.hexdigest(),
"counts": {
"videos": len(frozen),
"confirmed_negative_videos": confirmed_negatives,
"positive_videos_by_category": dict(sorted(positive_by_category.items())),
},
"output": {
"path": str(output_path),
"bytes": output_path.stat().st_size,
"sha256": sha256_file(output_path),
},
}
write_json_atomic(manifest_path, manifest)
return manifest