CuriousDragon's picture
Publish verified Flow Smart Segments curated-category-encoder-ensemble-20261004
8fcf7c9 verified
Raw History Blame Contribute Delete
38 kB
from __future__ import annotations
import csv
import hashlib
import json
import os
import tempfile
import time
from collections import Counter, defaultdict
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Callable, Iterable, Iterator, Sequence
from sponsor_detection.data.profile import (
EligibilityPolicy,
eligibility_rejection_flags,
sha256_file,
write_json_atomic,
)
from sponsor_detection.data.smart_segment_annotations import (
load_canonical_annotations,
)
from sponsor_detection.data.smart_segment_supervision import positive_supervision
from sponsor_detection.data.training_dataset import (
SPLITS,
DisjointSet,
_empty_batch,
_parquet_schema,
_split_for_component,
)
from sponsor_detection.inference.windowing import (
AssembledTranscript,
TranscriptCue,
TranscriptWindow,
assemble_transcript,
build_transcript_windows,
)
@dataclass(frozen=True)
class LabelSegment:
segment_id: str
start_ms: int
end_ms: int
category: str
def _clean_caption_line(text: str) -> str:
return " ".join(text.split())
def _longest_overlap_suffix(previous: str, text: str, *, maximum: int = 2000) -> str:
"""Return the portion of ``text`` not already covered by ``previous``.
YouTube auto-captions emit rolling cues where each cue repeats the tail of
the previous cue before appending new words. Removing the longest suffix of
``previous`` that is a prefix of ``text`` yields the newly spoken words once.
"""
limit = min(len(previous), len(text), maximum)
for length in range(limit, 0, -1):
if previous.endswith(text[:length]):
return text[length:].strip()
return text
def reconstruct_caption_lines(
cues: Iterable[tuple[int, int, str]],
) -> list[tuple[int, int, str]]:
"""Collapse rolling auto-caption cues into a deduplicated ordered line list."""
lines: list[list[object]] = []
previous = ""
for start_ms, end_ms, raw in cues:
text = _clean_caption_line(raw)
if not text:
continue
new_content = _longest_overlap_suffix(previous, text)
if not new_content:
previous = text
continue
if lines and new_content == previous:
previous = text
continue
lines.append([start_ms, end_ms, new_content])
previous = text
return [(int(start), int(end), str(text)) for start, end, text in lines]
def _load_metadata(directory: Path) -> dict[str, dict[str, object]]:
import pyarrow.parquet as parquet
metadata: dict[str, dict[str, object]] = {}
for path in sorted(directory.glob("*.parquet")):
file = parquet.ParquetFile(path)
columns = [
"id",
"channel_id",
"timestamp",
"upload_date",
"language",
"availability",
"live_status",
"duration",
]
for batch in file.iter_batches(columns=columns, batch_size=65_536):
values = [batch.column(column).to_pylist() for column in columns]
for (
video_id,
channel_id,
timestamp,
upload_date,
language,
availability,
live_status,
duration,
) in zip(*values, strict=True):
if video_id is None:
continue
metadata[str(video_id)] = {
"channel_id": channel_id,
"published_at": _published_at(timestamp, upload_date),
"language": language,
"availability": availability,
"live_status": live_status,
"duration": duration,
}
return metadata
def _published_at(timestamp: object, upload_date: object) -> str | None:
if isinstance(timestamp, int) and timestamp > 0:
return datetime.fromtimestamp(timestamp, tz=UTC).isoformat()
if isinstance(upload_date, str) and len(upload_date) == 8 and upload_date.isdigit():
return (
f"{upload_date[0:4]}-{upload_date[4:6]}-{upload_date[6:8]}T00:00:00+00:00"
)
return None
def _load_exclusions(
directories: Sequence[Path],
) -> tuple[set[str], set[str]]:
import pyarrow.parquet as parquet
videos: set[str] = set()
channels: set[str] = set()
for directory in directories:
if not directory.is_dir():
raise ValueError(f"exclusion directory does not exist: {directory}")
files = [directory / f"{split}.parquet" for split in SPLITS]
present = [path for path in files if path.is_file()]
if not present:
raise ValueError(
f"exclusion directory has no {SPLITS} parquet files: {directory}"
)
for path in present:
file = parquet.ParquetFile(path)
for batch in file.iter_batches(
columns=["video_id", "channel_id"], batch_size=65_536
):
videos.update(
str(value)
for value in batch.column("video_id").to_pylist()
if value is not None
)
channels.update(
str(value)
for value in batch.column("channel_id").to_pylist()
if value is not None
)
return videos, channels
def _load_labels(
mirror_path: Path,
relevant_video_ids: set[str],
*,
allowed_categories: set[str],
) -> dict[str, list[LabelSegment]]:
policy = EligibilityPolicy()
labels: dict[str, list[LabelSegment]] = defaultdict(list)
with mirror_path.open("r", encoding="utf-8", newline="") as source:
reader = csv.DictReader(source)
for row in reader:
video_id = row.get("videoID")
if video_id not in relevant_video_ids:
continue
category = row.get("category") or ""
if category not in allowed_categories:
continue
try:
start_time = float(row["startTime"])
end_time = float(row["endTime"])
votes = int(row["votes"])
hidden = int(row["hidden"])
shadow_hidden = int(row["shadowHidden"])
video_duration = float(row["videoDuration"] or 0.0)
except (KeyError, TypeError, ValueError):
continue
flags = eligibility_rejection_flags(
policy,
service=row.get("service") or "",
action_type=row.get("actionType") or "",
start_time=start_time,
end_time=end_time,
votes=votes,
hidden=hidden,
shadow_hidden=shadow_hidden,
video_duration=video_duration,
)
if flags:
continue
labels[video_id].append(
LabelSegment(
segment_id=str(row.get("UUID") or ""),
start_ms=round(start_time * 1000),
end_ms=round(end_time * 1000),
category=category,
)
)
for segments in labels.values():
segments.sort(key=lambda segment: (segment.start_ms, segment.end_ms))
return labels
def _char_span_for_interval(
transcript: AssembledTranscript, start_ms: int, end_ms: int
) -> tuple[int, int] | None:
start_char: int | None = None
end_char: int | None = None
for cue_range in transcript.cue_ranges:
if min(cue_range.end_ms, end_ms) > max(cue_range.start_ms, start_ms):
if start_char is None or cue_range.start_char < start_char:
start_char = cue_range.start_char
if end_char is None or cue_range.end_char > end_char:
end_char = cue_range.end_char
if start_char is None or end_char is None or end_char <= start_char:
return None
return start_char, end_char
def _window_spans_for_intervals(
transcript: AssembledTranscript,
window,
intervals_by_category: dict[str, list[tuple[int, int]]],
) -> list[dict[str, object]]:
spans: list[dict[str, object]] = []
for category, intervals in intervals_by_category.items():
for start_ms, end_ms in intervals:
char_span = _char_span_for_interval(transcript, start_ms, end_ms)
if char_span is None:
continue
local_start = max(char_span[0], window.start_char) - window.start_char
local_end = min(char_span[1], window.end_char) - window.start_char
if local_end <= local_start or local_start < 0:
continue
spans.append(
{
"start_char": local_start,
"end_char": local_end,
"start_ms": start_ms,
"end_ms": end_ms,
"current_segment_id": None,
"current_iou": None,
"campaign_hash": None,
"category": category,
}
)
spans.sort(key=lambda span: (str(span["category"]), int(span["start_char"])))
return spans
def _campaign_hash(text: str) -> str:
normalized = " ".join(text.lower().split())
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
def _is_english(language: str | None, prefixes: Sequence[str]) -> bool:
if not language:
return False
lowered = language.lower()
return any(lowered == prefix or lowered.startswith(f"{prefix}-") for prefix in prefixes)
def _iter_subtitle_rows(directory: Path) -> Iterator[dict[str, object]]:
import pyarrow.parquet as parquet
for path in sorted(directory.glob("*.parquet")):
file = parquet.ParquetFile(path)
for batch in file.iter_batches(
columns=["video_id", "language", "segments_json"], batch_size=512
):
records = batch.to_pydict()
for video_id, language, segments_json in zip(
records["video_id"],
records["language"],
records["segments_json"],
strict=True,
):
yield {
"video_id": str(video_id),
"language": language,
"segments_json": segments_json,
}
def _parse_cues(segments_json: str) -> list[tuple[int, int, str]]:
try:
payload = json.loads(segments_json)
except (TypeError, ValueError):
return []
cues: list[tuple[int, int, str]] = []
for cue in payload:
try:
start_ms = round(float(cue["start"]) * 1000)
end_ms = round(float(cue["end"]) * 1000)
text = str(cue["text"])
except (KeyError, TypeError, ValueError):
continue
if end_ms <= start_ms:
continue
cues.append((start_ms, end_ms, text))
cues.sort(key=lambda cue: (cue[0], cue[1]))
return cues
def _fit_window_to_max_tokens(
tokenizer, window: TranscriptWindow, max_length: int
) -> TranscriptWindow | None:
"""Trim a window so that re-tokenizing its text stays within ``max_length``.
``build_transcript_windows`` tokenizes the whole transcript at once, but the
trainer re-tokenizes each window independently. Byte-level merges across the
slice boundary can add a token or two, so windows are trimmed to the last
token boundary that a standalone encoding retains.
"""
text = window.text
for _ in range(max_length):
if not text:
return None
encoded = tokenizer(
text,
truncation=True,
max_length=max_length,
return_offsets_mapping=True,
)
content_offsets = [
(int(start), int(end))
for start, end in encoded["offset_mapping"]
if end > start
]
if not content_offsets:
return None
candidate = text[: content_offsets[-1][1]].rstrip()
if not candidate:
return None
if len(tokenizer(candidate, truncation=False)["input_ids"]) <= max_length:
if candidate == window.text:
return window
return replace(
window,
text=candidate,
end_char=window.start_char + len(candidate),
)
text = candidate[:-1]
return None
def build_scriptsmith_dataset(
subtitles_directory: Path,
metadata_directory: Path,
mirror_path: Path,
exclude_split_directories: Sequence[Path],
output_directory: Path,
manifest_path: Path,
*,
encoder: str,
encoder_revision: str,
max_length: int,
overlap_tokens: int,
seed: str,
train_fraction: float,
validation_fraction: float,
language_prefixes: Sequence[str],
positive_categories: Sequence[str],
hard_negative_categories: Sequence[str],
ordinary_negative_windows_per_video: int,
maximum_video_duration_seconds: int,
minimum_cues: int,
ordinary_negative_weight: float = 0.5,
provenance_path: Path | None = None,
maximum_videos: int = 0,
annotations_path: Path | None = None,
training_evidence_path: Path | None = None,
tokenizer: object | None = None,
progress_callback: Callable[[str], None] | None = None,
) -> dict[str, object]:
if not 0 < train_fraction < 1:
raise ValueError("train fraction must be between zero and one")
if not 0 < validation_fraction < 1 - train_fraction:
raise ValueError("validation fraction leaves no room for the test split")
if not 0 <= ordinary_negative_windows_per_video:
raise ValueError("ordinary negative cap must not be negative")
if maximum_video_duration_seconds <= 0:
raise ValueError("maximum video duration must be positive")
if minimum_cues <= 0:
raise ValueError("minimum cue count must be positive")
from transformers import AutoTokenizer
started = time.monotonic()
positive = set(positive_categories)
hard_negative = set(hard_negative_categories)
allowed = positive | hard_negative
if progress_callback:
progress_callback("loading metadata")
metadata = _load_metadata(metadata_directory)
if maximum_videos:
metadata = dict(list(metadata.items())[:maximum_videos])
if progress_callback:
progress_callback("loading exclusions from existing datasets")
excluded_videos, excluded_channels = _load_exclusions(exclude_split_directories)
from sponsor_detection.data.smart_segment_supervision import (
NEGATIVE_CONFIRMED,
NEGATIVE_WEAK,
load_training_evidence,
)
evidence = load_training_evidence(training_evidence_path)
negatives_by_video: dict[str, dict[str, list[tuple[int, int]]]] = defaultdict(
lambda: defaultdict(list)
)
weak_by_video: dict[str, dict[str, list[tuple[int, int]]]] = defaultdict(
lambda: defaultdict(list)
)
for record in evidence:
if record.state == NEGATIVE_CONFIRMED:
negatives_by_video[record.video_id][record.category].append(
(record.start_ms, record.end_ms)
)
elif record.state == NEGATIVE_WEAK:
weak_by_video[record.video_id][record.category].append(
(record.start_ms, record.end_ms)
)
def _excluded_reason(video_id: str) -> str | None:
if video_id in excluded_videos:
return "excluded_video"
channel_id = metadata.get(video_id, {}).get("channel_id")
if channel_id and str(channel_id) in excluded_channels:
return "excluded_channel"
return None
relevant_ids = {
video_id for video_id in metadata if _excluded_reason(video_id) is None
}
if progress_callback:
progress_callback(f"loading labels for {len(relevant_ids):,} videos")
if annotations_path is not None:
canonical = load_canonical_annotations(
annotations_path,
video_ids=relevant_ids,
categories=sorted(allowed),
eligible_only=True,
)
labels = {
video_id: [
LabelSegment(
segment_id=annotation.segment_id,
start_ms=annotation.start_ms,
end_ms=annotation.end_ms,
category=annotation.category,
)
for annotation in records
]
for video_id, records in canonical.items()
}
else:
labels = _load_labels(
mirror_path,
relevant_ids,
allowed_categories=allowed,
)
for video_id in relevant_ids:
labels.setdefault(video_id, [])
if progress_callback:
progress_callback("loading tokenizer")
if tokenizer is None:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(encoder, revision=encoder_revision)
funnel: Counter[str] = Counter()
accepted_videos: set[str] = set()
campaigns_by_video: dict[str, set[str]] = defaultdict(set)
label_kind_counts: Counter[str] = Counter()
output_directory.mkdir(parents=True, exist_ok=True)
temporary_directory = Path(
tempfile.mkdtemp(prefix=".scriptsmith-windows-", dir=output_directory)
)
all_records_path = temporary_directory / "all.parquet"
schema_metadata = {
b"sponsor_detection.schema_version": b"1",
b"sponsor_detection.dataset_revision": b"scriptsmith-2024",
b"sponsor_detection.split_seed": seed.encode(),
}
schema = _parquet_schema(schema_metadata)
writer = None
batch = _empty_batch()
video_index = 0
def flush_all() -> None:
nonlocal writer
import pyarrow as pa
import pyarrow.parquet as parquet
if not batch["video_id"]:
return
if writer is None:
writer = parquet.ParquetWriter(
all_records_path, schema, compression="zstd", use_dictionary=True
)
writer.write_table(pa.Table.from_pydict(batch, schema=schema))
for key in batch:
batch[key] = []
if progress_callback:
progress_callback("building windows")
try:
for row in _iter_subtitle_rows(subtitles_directory):
video_id = str(row["video_id"])
if video_id not in relevant_ids:
reason = _excluded_reason(video_id)
funnel[reason or "not_in_metadata"] += 1
continue
if not _is_english(row["language"], language_prefixes):
funnel["non_english"] += 1
continue
meta = metadata.get(video_id, {})
availability = meta.get("availability")
if availability is not None and availability != "public":
funnel["not_public"] += 1
continue
if meta.get("live_status") == "was_live":
funnel["was_live"] += 1
continue
duration = meta.get("duration")
if isinstance(duration, int) and duration > maximum_video_duration_seconds:
funnel["too_long"] += 1
continue
channel_id = meta.get("channel_id")
if not channel_id:
funnel["no_channel"] += 1
continue
raw_cues = _parse_cues(str(row["segments_json"]))
if len(raw_cues) < minimum_cues:
funnel["too_few_cues"] += 1
continue
reconstructed = reconstruct_caption_lines(raw_cues)
if not reconstructed:
funnel["empty_reconstruction"] += 1
continue
cues = [
TranscriptCue(index=index, start_ms=start, end_ms=end, text=text)
for index, (start, end, text) in enumerate(reconstructed)
]
transcript = assemble_transcript(cues)
if not transcript.text.strip():
funnel["empty_transcript"] += 1
continue
video_labels = labels.get(video_id, [])
positive_segments = [s for s in video_labels if s.category in positive]
hard_segments = [s for s in video_labels if s.category in hard_negative]
windows = []
for raw_window in build_transcript_windows(
transcript,
tokenizer,
max_length=max_length,
overlap_tokens=overlap_tokens,
):
fitted = _fit_window_to_max_tokens(tokenizer, raw_window, max_length)
if fitted is not None:
windows.append(fitted)
if not windows:
funnel["no_windows"] += 1
continue
video_accepted = False
ordinary_emitted = 0
for window in windows:
window_spans = []
for segment in positive_segments:
char_span = _char_span_for_interval(
transcript, segment.start_ms, segment.end_ms
)
if char_span is None:
continue
local_start = max(char_span[0], window.start_char) - window.start_char
local_end = min(char_span[1], window.end_char) - window.start_char
if local_end <= local_start or local_start < 0:
continue
text = transcript.text[char_span[0] : char_span[1]]
window_spans.append(
{
"start_char": local_start,
"end_char": local_end,
"start_ms": segment.start_ms,
"end_ms": segment.end_ms,
"current_segment_id": segment.segment_id,
"current_iou": 1.0,
"campaign_hash": _campaign_hash(text),
"category": segment.category,
}
)
if window_spans:
window_spans.sort(
key=lambda span: (
str(span["category"]),
span["start_char"],
span["end_char"],
)
)
deduped: list[dict[str, object]] = []
last_end_by_category: dict[str, int] = {}
for span in window_spans:
category = str(span["category"])
last_end = last_end_by_category.get(category, -1)
if int(span["start_char"]) < last_end:
continue
deduped.append(span)
last_end_by_category[category] = int(span["end_char"])
window_spans = deduped
label_kind = "positive"
categories = sorted(
{str(span["category"]).upper() for span in window_spans}
)
sample_weight = 1.0
else:
overlaps_hard = any(
min(segment.end_ms, window.end_ms)
> max(segment.start_ms, window.start_ms)
for segment in hard_segments
)
if overlaps_hard:
label_kind = "hard_negative"
categories = sorted({c.upper() for c in hard_negative})
sample_weight = 1.0
else:
if ordinary_emitted >= ordinary_negative_windows_per_video:
continue
ordinary_emitted += 1
label_kind = "ordinary_negative"
categories = []
sample_weight = ordinary_negative_weight
sponsor_spans = [
span for span in window_spans if span["category"] == "sponsor"
]
supervision_evidence: dict[str, str] = {}
for span in window_spans:
supervision_evidence.setdefault(
str(span["category"]), str(span["current_segment_id"])
)
identity = (
f"{video_id}\0{window.start_ms}\0{window.end_ms}\0{window.text}"
)
record = {
"example_id": hashlib.sha256(identity.encode("utf-8")).hexdigest(),
"video_index": video_index,
"video_id": video_id,
"channel_id": channel_id,
"published_at": meta.get("published_at"),
"source_split": "scriptsmith",
"text": window.text,
"window_start_ms": window.start_ms,
"window_end_ms": window.end_ms,
"label_kind": label_kind,
"sample_weight": sample_weight,
"legacy_categories": categories,
"sponsor_spans": sponsor_spans,
"category_spans": window_spans,
"negative_spans": _window_spans_for_intervals(
transcript, window, negatives_by_video.get(video_id, {})
),
"weak_negative_spans": _window_spans_for_intervals(
transcript, window, weak_by_video.get(video_id, {})
),
"category_supervision": positive_supervision(
supervision_evidence
),
}
for field in batch:
batch[field].append(record[field])
label_kind_counts[label_kind] += 1
videos_by_label = campaigns_by_video.setdefault(video_id, set())
for span in sponsor_spans:
videos_by_label.add(str(span["campaign_hash"]))
video_accepted = True
if len(batch["video_id"]) >= 8192:
flush_all()
if video_accepted:
accepted_videos.add(video_id)
else:
funnel["no_accepted_windows"] += 1
video_index += 1
if progress_callback and video_index % 1000 == 0:
progress_callback(
f"processed {video_index:,} videos, {len(accepted_videos):,} accepted"
)
flush_all()
if writer is not None:
writer.close()
writer = None
if not all_records_path.exists():
import pyarrow as pa
import pyarrow.parquet as parquet
parquet.write_table(
pa.Table.from_pydict(_empty_batch(), schema=schema),
all_records_path,
compression="zstd",
)
if progress_callback:
progress_callback("splitting channel-disjoint")
groups = DisjointSet(accepted_videos)
first_by_channel: dict[str, str] = {}
first_by_campaign: dict[str, str] = {}
for video_id in sorted(accepted_videos):
channel_id = metadata.get(video_id, {}).get("channel_id")
if channel_id:
previous = first_by_channel.setdefault(str(channel_id), video_id)
groups.union(previous, video_id)
for campaign_hash in campaigns_by_video.get(video_id, ()):
previous = first_by_campaign.setdefault(campaign_hash, video_id)
groups.union(previous, video_id)
component_members: dict[str, list[str]] = defaultdict(list)
for video_id in accepted_videos:
component_members[groups.find(video_id)].append(video_id)
component_sizes = sorted(len(members) for members in component_members.values())
def component_percentile(fraction: float) -> int:
if not component_sizes:
return 0
return component_sizes[round((len(component_sizes) - 1) * fraction)]
split_by_video: dict[str, str] = {}
for members in component_members.values():
split = _split_for_component(
min(members),
seed=seed,
train_fraction=train_fraction,
validation_fraction=validation_fraction,
)
for video_id in members:
split_by_video[video_id] = split
if progress_callback:
progress_callback("routing split outputs")
import pyarrow.parquet as parquet
split_batches = {split: _empty_batch() for split in SPLITS}
split_writers: dict[str, object] = {}
counts_by_split = {split: Counter() for split in SPLITS}
videos_by_split = {split: set() for split in SPLITS}
channels_by_split = {split: set() for split in SPLITS}
campaigns_by_split = {split: set() for split in SPLITS}
def flush_split(split: str) -> None:
import pyarrow as pa
split_batch = split_batches[split]
if not split_batch["video_id"]:
return
split_writer = split_writers.get(split)
if split_writer is None:
split_writer = parquet.ParquetWriter(
temporary_directory / f"{split}.parquet",
schema,
compression="zstd",
use_dictionary=True,
)
split_writers[split] = split_writer
split_writer.write_table(pa.Table.from_pydict(split_batch, schema=schema))
for key in split_batch:
split_batch[key] = []
all_file = parquet.ParquetFile(all_records_path)
for record_batch in all_file.iter_batches():
records = record_batch.to_pydict()
for index in range(len(records["video_id"])):
record = {key: values[index] for key, values in records.items()}
split = split_by_video.get(str(record["video_id"]))
if split is None:
continue
split_batch = split_batches[split]
for field in split_batch:
split_batch[field].append(record[field])
counts_by_split[split][str(record["label_kind"])] += 1
videos_by_split[split].add(str(record["video_id"]))
if record["channel_id"]:
channels_by_split[split].add(str(record["channel_id"]))
campaigns_by_split[split].update(
str(span["campaign_hash"]) for span in record["sponsor_spans"]
)
if len(split_batch["video_id"]) >= 8192:
flush_split(split)
for split in SPLITS:
flush_split(split)
for split_writer in split_writers.values():
split_writer.close()
import pyarrow as pa
for split in SPLITS:
temporary_path = temporary_directory / f"{split}.parquet"
if not temporary_path.exists():
parquet.write_table(
pa.Table.from_pydict(_empty_batch(), schema=schema),
temporary_path,
compression="zstd",
)
for split in SPLITS:
os.replace(
temporary_directory / f"{split}.parquet",
output_directory / f"{split}.parquet",
)
finally:
if writer is not None:
writer.close()
for path in temporary_directory.glob("*"):
path.unlink(missing_ok=True)
temporary_directory.rmdir()
leakage = {
"video": {
"train_validation": len(videos_by_split["train"] & videos_by_split["validation"]),
"train_test": len(videos_by_split["train"] & videos_by_split["test"]),
"validation_test": len(
videos_by_split["validation"] & videos_by_split["test"]
),
},
"channel": {
"train_validation": len(
channels_by_split["train"] & channels_by_split["validation"]
),
"train_test": len(channels_by_split["train"] & channels_by_split["test"]),
"validation_test": len(
channels_by_split["validation"] & channels_by_split["test"]
),
},
"exact_campaign": {
"train_validation": len(
campaigns_by_split["train"] & campaigns_by_split["validation"]
),
"train_test": len(campaigns_by_split["train"] & campaigns_by_split["test"]),
"validation_test": len(
campaigns_by_split["validation"] & campaigns_by_split["test"]
),
},
}
if any(
value
for category in leakage.values()
for value in category.values()
if value is not None
):
raise AssertionError(f"group leakage detected: {leakage}")
accepted_channels = {
str(metadata[video_id]["channel_id"])
for video_id in accepted_videos
if metadata.get(video_id, {}).get("channel_id")
}
reservations = {
"excluded_videos": len(excluded_videos),
"excluded_channels": len(excluded_channels),
"accepted_videos_in_excluded_videos": len(accepted_videos & excluded_videos),
"accepted_channels_in_excluded_channels": len(
accepted_channels & excluded_channels
),
}
if (
reservations["accepted_videos_in_excluded_videos"]
or reservations["accepted_channels_in_excluded_channels"]
):
raise AssertionError(f"reserved data leaked into the dataset: {reservations}")
for split in SPLITS:
funnel[f"split_{split}_rows"] = sum(counts_by_split[split].values())
outputs = {}
for split in SPLITS:
path = output_directory / f"{split}.parquet"
outputs[split] = {
"path": str(path),
"bytes": path.stat().st_size,
"sha256": sha256_file(path),
"rows": sum(counts_by_split[split].values()),
"videos": len(videos_by_split[split]),
"channels": len(channels_by_split[split]),
"campaigns": len(campaigns_by_split[split]),
"label_kinds": dict(counts_by_split[split]),
}
manifest = {
"schema_version": 1,
"generated_at": datetime.now(tz=UTC).isoformat(),
"configuration": {
"seed": seed,
"train_fraction": train_fraction,
"validation_fraction": validation_fraction,
"test_fraction": round(1 - train_fraction - validation_fraction, 6),
"encoder": encoder,
"encoder_revision": encoder_revision,
"max_length": max_length,
"overlap_tokens": overlap_tokens,
"language_prefixes": list(language_prefixes),
"positive_categories": sorted(positive),
"hard_negative_categories": sorted(hard_negative),
"ordinary_negative_windows_per_video": ordinary_negative_windows_per_video,
"maximum_video_duration_seconds": maximum_video_duration_seconds,
"minimum_cues": minimum_cues,
"exclude_split_directories": [str(path) for path in exclude_split_directories],
},
"sources": {
"subtitles_directory": str(subtitles_directory),
"metadata_directory": str(metadata_directory),
"mirror_path": str(mirror_path),
"annotations_path": str(annotations_path) if annotations_path else None,
"annotations_sha256": (
sha256_file(annotations_path)
if annotations_path is not None and annotations_path.exists()
else None
),
"training_evidence_path": (
str(training_evidence_path) if training_evidence_path else None
),
"training_evidence_sha256": (
sha256_file(training_evidence_path)
if training_evidence_path is not None
and training_evidence_path.exists()
else None
),
"provenance_path": str(provenance_path) if provenance_path else None,
"provenance_sha256": (
sha256_file(provenance_path)
if provenance_path is not None and provenance_path.exists()
else None
),
},
"exclusions": {
"excluded_videos": len(excluded_videos),
"excluded_channels": len(excluded_channels),
},
"reservations": reservations,
"funnel": dict(funnel),
"label_kinds": dict(label_kind_counts),
"groups": {
"components": len(component_members),
"accepted_videos": len(accepted_videos),
"channels": len(first_by_channel),
"exact_campaigns": len(first_by_campaign),
"component_size_videos": {
"p50": component_percentile(0.50),
"p95": component_percentile(0.95),
"p99": component_percentile(0.99),
"max": component_sizes[-1] if component_sizes else 0,
},
},
"leakage": leakage,
"outputs": outputs,
"elapsed_seconds": round(time.monotonic() - started, 3),
}
write_json_atomic(manifest_path, manifest)
return manifest