from __future__ import annotations import csv import hashlib import json import os import tempfile import time from collections import Counter, defaultdict from dataclasses import dataclass, replace from datetime import UTC, datetime from pathlib import Path from typing import Callable, Iterable, Iterator, Sequence from sponsor_detection.data.profile import ( EligibilityPolicy, eligibility_rejection_flags, sha256_file, write_json_atomic, ) from sponsor_detection.data.smart_segment_annotations import ( load_canonical_annotations, ) from sponsor_detection.data.smart_segment_supervision import positive_supervision from sponsor_detection.data.training_dataset import ( SPLITS, DisjointSet, _empty_batch, _parquet_schema, _split_for_component, ) from sponsor_detection.inference.windowing import ( AssembledTranscript, TranscriptCue, TranscriptWindow, assemble_transcript, build_transcript_windows, ) @dataclass(frozen=True) class LabelSegment: segment_id: str start_ms: int end_ms: int category: str def _clean_caption_line(text: str) -> str: return " ".join(text.split()) def _longest_overlap_suffix(previous: str, text: str, *, maximum: int = 2000) -> str: """Return the portion of ``text`` not already covered by ``previous``. YouTube auto-captions emit rolling cues where each cue repeats the tail of the previous cue before appending new words. Removing the longest suffix of ``previous`` that is a prefix of ``text`` yields the newly spoken words once. """ limit = min(len(previous), len(text), maximum) for length in range(limit, 0, -1): if previous.endswith(text[:length]): return text[length:].strip() return text def reconstruct_caption_lines( cues: Iterable[tuple[int, int, str]], ) -> list[tuple[int, int, str]]: """Collapse rolling auto-caption cues into a deduplicated ordered line list.""" lines: list[list[object]] = [] previous = "" for start_ms, end_ms, raw in cues: text = _clean_caption_line(raw) if not text: continue new_content = _longest_overlap_suffix(previous, text) if not new_content: previous = text continue if lines and new_content == previous: previous = text continue lines.append([start_ms, end_ms, new_content]) previous = text return [(int(start), int(end), str(text)) for start, end, text in lines] def _load_metadata(directory: Path) -> dict[str, dict[str, object]]: import pyarrow.parquet as parquet metadata: dict[str, dict[str, object]] = {} for path in sorted(directory.glob("*.parquet")): file = parquet.ParquetFile(path) columns = [ "id", "channel_id", "timestamp", "upload_date", "language", "availability", "live_status", "duration", ] for batch in file.iter_batches(columns=columns, batch_size=65_536): values = [batch.column(column).to_pylist() for column in columns] for ( video_id, channel_id, timestamp, upload_date, language, availability, live_status, duration, ) in zip(*values, strict=True): if video_id is None: continue metadata[str(video_id)] = { "channel_id": channel_id, "published_at": _published_at(timestamp, upload_date), "language": language, "availability": availability, "live_status": live_status, "duration": duration, } return metadata def _published_at(timestamp: object, upload_date: object) -> str | None: if isinstance(timestamp, int) and timestamp > 0: return datetime.fromtimestamp(timestamp, tz=UTC).isoformat() if isinstance(upload_date, str) and len(upload_date) == 8 and upload_date.isdigit(): return ( f"{upload_date[0:4]}-{upload_date[4:6]}-{upload_date[6:8]}T00:00:00+00:00" ) return None def _load_exclusions( directories: Sequence[Path], ) -> tuple[set[str], set[str]]: import pyarrow.parquet as parquet videos: set[str] = set() channels: set[str] = set() for directory in directories: if not directory.is_dir(): raise ValueError(f"exclusion directory does not exist: {directory}") files = [directory / f"{split}.parquet" for split in SPLITS] present = [path for path in files if path.is_file()] if not present: raise ValueError( f"exclusion directory has no {SPLITS} parquet files: {directory}" ) for path in present: file = parquet.ParquetFile(path) for batch in file.iter_batches( columns=["video_id", "channel_id"], batch_size=65_536 ): videos.update( str(value) for value in batch.column("video_id").to_pylist() if value is not None ) channels.update( str(value) for value in batch.column("channel_id").to_pylist() if value is not None ) return videos, channels def _load_labels( mirror_path: Path, relevant_video_ids: set[str], *, allowed_categories: set[str], ) -> dict[str, list[LabelSegment]]: policy = EligibilityPolicy() labels: dict[str, list[LabelSegment]] = defaultdict(list) with mirror_path.open("r", encoding="utf-8", newline="") as source: reader = csv.DictReader(source) for row in reader: video_id = row.get("videoID") if video_id not in relevant_video_ids: continue category = row.get("category") or "" if category not in allowed_categories: continue try: start_time = float(row["startTime"]) end_time = float(row["endTime"]) votes = int(row["votes"]) hidden = int(row["hidden"]) shadow_hidden = int(row["shadowHidden"]) video_duration = float(row["videoDuration"] or 0.0) except (KeyError, TypeError, ValueError): continue flags = eligibility_rejection_flags( policy, service=row.get("service") or "", action_type=row.get("actionType") or "", start_time=start_time, end_time=end_time, votes=votes, hidden=hidden, shadow_hidden=shadow_hidden, video_duration=video_duration, ) if flags: continue labels[video_id].append( LabelSegment( segment_id=str(row.get("UUID") or ""), start_ms=round(start_time * 1000), end_ms=round(end_time * 1000), category=category, ) ) for segments in labels.values(): segments.sort(key=lambda segment: (segment.start_ms, segment.end_ms)) return labels def _char_span_for_interval( transcript: AssembledTranscript, start_ms: int, end_ms: int ) -> tuple[int, int] | None: start_char: int | None = None end_char: int | None = None for cue_range in transcript.cue_ranges: if min(cue_range.end_ms, end_ms) > max(cue_range.start_ms, start_ms): if start_char is None or cue_range.start_char < start_char: start_char = cue_range.start_char if end_char is None or cue_range.end_char > end_char: end_char = cue_range.end_char if start_char is None or end_char is None or end_char <= start_char: return None return start_char, end_char def _window_spans_for_intervals( transcript: AssembledTranscript, window, intervals_by_category: dict[str, list[tuple[int, int]]], ) -> list[dict[str, object]]: spans: list[dict[str, object]] = [] for category, intervals in intervals_by_category.items(): for start_ms, end_ms in intervals: char_span = _char_span_for_interval(transcript, start_ms, end_ms) if char_span is None: continue local_start = max(char_span[0], window.start_char) - window.start_char local_end = min(char_span[1], window.end_char) - window.start_char if local_end <= local_start or local_start < 0: continue spans.append( { "start_char": local_start, "end_char": local_end, "start_ms": start_ms, "end_ms": end_ms, "current_segment_id": None, "current_iou": None, "campaign_hash": None, "category": category, } ) spans.sort(key=lambda span: (str(span["category"]), int(span["start_char"]))) return spans def _campaign_hash(text: str) -> str: normalized = " ".join(text.lower().split()) return hashlib.sha256(normalized.encode("utf-8")).hexdigest() def _is_english(language: str | None, prefixes: Sequence[str]) -> bool: if not language: return False lowered = language.lower() return any(lowered == prefix or lowered.startswith(f"{prefix}-") for prefix in prefixes) def _iter_subtitle_rows(directory: Path) -> Iterator[dict[str, object]]: import pyarrow.parquet as parquet for path in sorted(directory.glob("*.parquet")): file = parquet.ParquetFile(path) for batch in file.iter_batches( columns=["video_id", "language", "segments_json"], batch_size=512 ): records = batch.to_pydict() for video_id, language, segments_json in zip( records["video_id"], records["language"], records["segments_json"], strict=True, ): yield { "video_id": str(video_id), "language": language, "segments_json": segments_json, } def _parse_cues(segments_json: str) -> list[tuple[int, int, str]]: try: payload = json.loads(segments_json) except (TypeError, ValueError): return [] cues: list[tuple[int, int, str]] = [] for cue in payload: try: start_ms = round(float(cue["start"]) * 1000) end_ms = round(float(cue["end"]) * 1000) text = str(cue["text"]) except (KeyError, TypeError, ValueError): continue if end_ms <= start_ms: continue cues.append((start_ms, end_ms, text)) cues.sort(key=lambda cue: (cue[0], cue[1])) return cues def _fit_window_to_max_tokens( tokenizer, window: TranscriptWindow, max_length: int ) -> TranscriptWindow | None: """Trim a window so that re-tokenizing its text stays within ``max_length``. ``build_transcript_windows`` tokenizes the whole transcript at once, but the trainer re-tokenizes each window independently. Byte-level merges across the slice boundary can add a token or two, so windows are trimmed to the last token boundary that a standalone encoding retains. """ text = window.text for _ in range(max_length): if not text: return None encoded = tokenizer( text, truncation=True, max_length=max_length, return_offsets_mapping=True, ) content_offsets = [ (int(start), int(end)) for start, end in encoded["offset_mapping"] if end > start ] if not content_offsets: return None candidate = text[: content_offsets[-1][1]].rstrip() if not candidate: return None if len(tokenizer(candidate, truncation=False)["input_ids"]) <= max_length: if candidate == window.text: return window return replace( window, text=candidate, end_char=window.start_char + len(candidate), ) text = candidate[:-1] return None def build_scriptsmith_dataset( subtitles_directory: Path, metadata_directory: Path, mirror_path: Path, exclude_split_directories: Sequence[Path], output_directory: Path, manifest_path: Path, *, encoder: str, encoder_revision: str, max_length: int, overlap_tokens: int, seed: str, train_fraction: float, validation_fraction: float, language_prefixes: Sequence[str], positive_categories: Sequence[str], hard_negative_categories: Sequence[str], ordinary_negative_windows_per_video: int, maximum_video_duration_seconds: int, minimum_cues: int, ordinary_negative_weight: float = 0.5, provenance_path: Path | None = None, maximum_videos: int = 0, annotations_path: Path | None = None, training_evidence_path: Path | None = None, tokenizer: object | None = None, progress_callback: Callable[[str], None] | None = None, ) -> dict[str, object]: if not 0 < train_fraction < 1: raise ValueError("train fraction must be between zero and one") if not 0 < validation_fraction < 1 - train_fraction: raise ValueError("validation fraction leaves no room for the test split") if not 0 <= ordinary_negative_windows_per_video: raise ValueError("ordinary negative cap must not be negative") if maximum_video_duration_seconds <= 0: raise ValueError("maximum video duration must be positive") if minimum_cues <= 0: raise ValueError("minimum cue count must be positive") from transformers import AutoTokenizer started = time.monotonic() positive = set(positive_categories) hard_negative = set(hard_negative_categories) allowed = positive | hard_negative if progress_callback: progress_callback("loading metadata") metadata = _load_metadata(metadata_directory) if maximum_videos: metadata = dict(list(metadata.items())[:maximum_videos]) if progress_callback: progress_callback("loading exclusions from existing datasets") excluded_videos, excluded_channels = _load_exclusions(exclude_split_directories) from sponsor_detection.data.smart_segment_supervision import ( NEGATIVE_CONFIRMED, NEGATIVE_WEAK, load_training_evidence, ) evidence = load_training_evidence(training_evidence_path) negatives_by_video: dict[str, dict[str, list[tuple[int, int]]]] = defaultdict( lambda: defaultdict(list) ) weak_by_video: dict[str, dict[str, list[tuple[int, int]]]] = defaultdict( lambda: defaultdict(list) ) for record in evidence: if record.state == NEGATIVE_CONFIRMED: negatives_by_video[record.video_id][record.category].append( (record.start_ms, record.end_ms) ) elif record.state == NEGATIVE_WEAK: weak_by_video[record.video_id][record.category].append( (record.start_ms, record.end_ms) ) def _excluded_reason(video_id: str) -> str | None: if video_id in excluded_videos: return "excluded_video" channel_id = metadata.get(video_id, {}).get("channel_id") if channel_id and str(channel_id) in excluded_channels: return "excluded_channel" return None relevant_ids = { video_id for video_id in metadata if _excluded_reason(video_id) is None } if progress_callback: progress_callback(f"loading labels for {len(relevant_ids):,} videos") if annotations_path is not None: canonical = load_canonical_annotations( annotations_path, video_ids=relevant_ids, categories=sorted(allowed), eligible_only=True, ) labels = { video_id: [ LabelSegment( segment_id=annotation.segment_id, start_ms=annotation.start_ms, end_ms=annotation.end_ms, category=annotation.category, ) for annotation in records ] for video_id, records in canonical.items() } else: labels = _load_labels( mirror_path, relevant_ids, allowed_categories=allowed, ) for video_id in relevant_ids: labels.setdefault(video_id, []) if progress_callback: progress_callback("loading tokenizer") if tokenizer is None: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(encoder, revision=encoder_revision) funnel: Counter[str] = Counter() accepted_videos: set[str] = set() campaigns_by_video: dict[str, set[str]] = defaultdict(set) label_kind_counts: Counter[str] = Counter() output_directory.mkdir(parents=True, exist_ok=True) temporary_directory = Path( tempfile.mkdtemp(prefix=".scriptsmith-windows-", dir=output_directory) ) all_records_path = temporary_directory / "all.parquet" schema_metadata = { b"sponsor_detection.schema_version": b"1", b"sponsor_detection.dataset_revision": b"scriptsmith-2024", b"sponsor_detection.split_seed": seed.encode(), } schema = _parquet_schema(schema_metadata) writer = None batch = _empty_batch() video_index = 0 def flush_all() -> None: nonlocal writer import pyarrow as pa import pyarrow.parquet as parquet if not batch["video_id"]: return if writer is None: writer = parquet.ParquetWriter( all_records_path, schema, compression="zstd", use_dictionary=True ) writer.write_table(pa.Table.from_pydict(batch, schema=schema)) for key in batch: batch[key] = [] if progress_callback: progress_callback("building windows") try: for row in _iter_subtitle_rows(subtitles_directory): video_id = str(row["video_id"]) if video_id not in relevant_ids: reason = _excluded_reason(video_id) funnel[reason or "not_in_metadata"] += 1 continue if not _is_english(row["language"], language_prefixes): funnel["non_english"] += 1 continue meta = metadata.get(video_id, {}) availability = meta.get("availability") if availability is not None and availability != "public": funnel["not_public"] += 1 continue if meta.get("live_status") == "was_live": funnel["was_live"] += 1 continue duration = meta.get("duration") if isinstance(duration, int) and duration > maximum_video_duration_seconds: funnel["too_long"] += 1 continue channel_id = meta.get("channel_id") if not channel_id: funnel["no_channel"] += 1 continue raw_cues = _parse_cues(str(row["segments_json"])) if len(raw_cues) < minimum_cues: funnel["too_few_cues"] += 1 continue reconstructed = reconstruct_caption_lines(raw_cues) if not reconstructed: funnel["empty_reconstruction"] += 1 continue cues = [ TranscriptCue(index=index, start_ms=start, end_ms=end, text=text) for index, (start, end, text) in enumerate(reconstructed) ] transcript = assemble_transcript(cues) if not transcript.text.strip(): funnel["empty_transcript"] += 1 continue video_labels = labels.get(video_id, []) positive_segments = [s for s in video_labels if s.category in positive] hard_segments = [s for s in video_labels if s.category in hard_negative] windows = [] for raw_window in build_transcript_windows( transcript, tokenizer, max_length=max_length, overlap_tokens=overlap_tokens, ): fitted = _fit_window_to_max_tokens(tokenizer, raw_window, max_length) if fitted is not None: windows.append(fitted) if not windows: funnel["no_windows"] += 1 continue video_accepted = False ordinary_emitted = 0 for window in windows: window_spans = [] for segment in positive_segments: char_span = _char_span_for_interval( transcript, segment.start_ms, segment.end_ms ) if char_span is None: continue local_start = max(char_span[0], window.start_char) - window.start_char local_end = min(char_span[1], window.end_char) - window.start_char if local_end <= local_start or local_start < 0: continue text = transcript.text[char_span[0] : char_span[1]] window_spans.append( { "start_char": local_start, "end_char": local_end, "start_ms": segment.start_ms, "end_ms": segment.end_ms, "current_segment_id": segment.segment_id, "current_iou": 1.0, "campaign_hash": _campaign_hash(text), "category": segment.category, } ) if window_spans: window_spans.sort( key=lambda span: ( str(span["category"]), span["start_char"], span["end_char"], ) ) deduped: list[dict[str, object]] = [] last_end_by_category: dict[str, int] = {} for span in window_spans: category = str(span["category"]) last_end = last_end_by_category.get(category, -1) if int(span["start_char"]) < last_end: continue deduped.append(span) last_end_by_category[category] = int(span["end_char"]) window_spans = deduped label_kind = "positive" categories = sorted( {str(span["category"]).upper() for span in window_spans} ) sample_weight = 1.0 else: overlaps_hard = any( min(segment.end_ms, window.end_ms) > max(segment.start_ms, window.start_ms) for segment in hard_segments ) if overlaps_hard: label_kind = "hard_negative" categories = sorted({c.upper() for c in hard_negative}) sample_weight = 1.0 else: if ordinary_emitted >= ordinary_negative_windows_per_video: continue ordinary_emitted += 1 label_kind = "ordinary_negative" categories = [] sample_weight = ordinary_negative_weight sponsor_spans = [ span for span in window_spans if span["category"] == "sponsor" ] supervision_evidence: dict[str, str] = {} for span in window_spans: supervision_evidence.setdefault( str(span["category"]), str(span["current_segment_id"]) ) identity = ( f"{video_id}\0{window.start_ms}\0{window.end_ms}\0{window.text}" ) record = { "example_id": hashlib.sha256(identity.encode("utf-8")).hexdigest(), "video_index": video_index, "video_id": video_id, "channel_id": channel_id, "published_at": meta.get("published_at"), "source_split": "scriptsmith", "text": window.text, "window_start_ms": window.start_ms, "window_end_ms": window.end_ms, "label_kind": label_kind, "sample_weight": sample_weight, "legacy_categories": categories, "sponsor_spans": sponsor_spans, "category_spans": window_spans, "negative_spans": _window_spans_for_intervals( transcript, window, negatives_by_video.get(video_id, {}) ), "weak_negative_spans": _window_spans_for_intervals( transcript, window, weak_by_video.get(video_id, {}) ), "category_supervision": positive_supervision( supervision_evidence ), } for field in batch: batch[field].append(record[field]) label_kind_counts[label_kind] += 1 videos_by_label = campaigns_by_video.setdefault(video_id, set()) for span in sponsor_spans: videos_by_label.add(str(span["campaign_hash"])) video_accepted = True if len(batch["video_id"]) >= 8192: flush_all() if video_accepted: accepted_videos.add(video_id) else: funnel["no_accepted_windows"] += 1 video_index += 1 if progress_callback and video_index % 1000 == 0: progress_callback( f"processed {video_index:,} videos, {len(accepted_videos):,} accepted" ) flush_all() if writer is not None: writer.close() writer = None if not all_records_path.exists(): import pyarrow as pa import pyarrow.parquet as parquet parquet.write_table( pa.Table.from_pydict(_empty_batch(), schema=schema), all_records_path, compression="zstd", ) if progress_callback: progress_callback("splitting channel-disjoint") groups = DisjointSet(accepted_videos) first_by_channel: dict[str, str] = {} first_by_campaign: dict[str, str] = {} for video_id in sorted(accepted_videos): channel_id = metadata.get(video_id, {}).get("channel_id") if channel_id: previous = first_by_channel.setdefault(str(channel_id), video_id) groups.union(previous, video_id) for campaign_hash in campaigns_by_video.get(video_id, ()): previous = first_by_campaign.setdefault(campaign_hash, video_id) groups.union(previous, video_id) component_members: dict[str, list[str]] = defaultdict(list) for video_id in accepted_videos: component_members[groups.find(video_id)].append(video_id) component_sizes = sorted(len(members) for members in component_members.values()) def component_percentile(fraction: float) -> int: if not component_sizes: return 0 return component_sizes[round((len(component_sizes) - 1) * fraction)] split_by_video: dict[str, str] = {} for members in component_members.values(): split = _split_for_component( min(members), seed=seed, train_fraction=train_fraction, validation_fraction=validation_fraction, ) for video_id in members: split_by_video[video_id] = split if progress_callback: progress_callback("routing split outputs") import pyarrow.parquet as parquet split_batches = {split: _empty_batch() for split in SPLITS} split_writers: dict[str, object] = {} counts_by_split = {split: Counter() for split in SPLITS} videos_by_split = {split: set() for split in SPLITS} channels_by_split = {split: set() for split in SPLITS} campaigns_by_split = {split: set() for split in SPLITS} def flush_split(split: str) -> None: import pyarrow as pa split_batch = split_batches[split] if not split_batch["video_id"]: return split_writer = split_writers.get(split) if split_writer is None: split_writer = parquet.ParquetWriter( temporary_directory / f"{split}.parquet", schema, compression="zstd", use_dictionary=True, ) split_writers[split] = split_writer split_writer.write_table(pa.Table.from_pydict(split_batch, schema=schema)) for key in split_batch: split_batch[key] = [] all_file = parquet.ParquetFile(all_records_path) for record_batch in all_file.iter_batches(): records = record_batch.to_pydict() for index in range(len(records["video_id"])): record = {key: values[index] for key, values in records.items()} split = split_by_video.get(str(record["video_id"])) if split is None: continue split_batch = split_batches[split] for field in split_batch: split_batch[field].append(record[field]) counts_by_split[split][str(record["label_kind"])] += 1 videos_by_split[split].add(str(record["video_id"])) if record["channel_id"]: channels_by_split[split].add(str(record["channel_id"])) campaigns_by_split[split].update( str(span["campaign_hash"]) for span in record["sponsor_spans"] ) if len(split_batch["video_id"]) >= 8192: flush_split(split) for split in SPLITS: flush_split(split) for split_writer in split_writers.values(): split_writer.close() import pyarrow as pa for split in SPLITS: temporary_path = temporary_directory / f"{split}.parquet" if not temporary_path.exists(): parquet.write_table( pa.Table.from_pydict(_empty_batch(), schema=schema), temporary_path, compression="zstd", ) for split in SPLITS: os.replace( temporary_directory / f"{split}.parquet", output_directory / f"{split}.parquet", ) finally: if writer is not None: writer.close() for path in temporary_directory.glob("*"): path.unlink(missing_ok=True) temporary_directory.rmdir() leakage = { "video": { "train_validation": len(videos_by_split["train"] & videos_by_split["validation"]), "train_test": len(videos_by_split["train"] & videos_by_split["test"]), "validation_test": len( videos_by_split["validation"] & videos_by_split["test"] ), }, "channel": { "train_validation": len( channels_by_split["train"] & channels_by_split["validation"] ), "train_test": len(channels_by_split["train"] & channels_by_split["test"]), "validation_test": len( channels_by_split["validation"] & channels_by_split["test"] ), }, "exact_campaign": { "train_validation": len( campaigns_by_split["train"] & campaigns_by_split["validation"] ), "train_test": len(campaigns_by_split["train"] & campaigns_by_split["test"]), "validation_test": len( campaigns_by_split["validation"] & campaigns_by_split["test"] ), }, } if any( value for category in leakage.values() for value in category.values() if value is not None ): raise AssertionError(f"group leakage detected: {leakage}") accepted_channels = { str(metadata[video_id]["channel_id"]) for video_id in accepted_videos if metadata.get(video_id, {}).get("channel_id") } reservations = { "excluded_videos": len(excluded_videos), "excluded_channels": len(excluded_channels), "accepted_videos_in_excluded_videos": len(accepted_videos & excluded_videos), "accepted_channels_in_excluded_channels": len( accepted_channels & excluded_channels ), } if ( reservations["accepted_videos_in_excluded_videos"] or reservations["accepted_channels_in_excluded_channels"] ): raise AssertionError(f"reserved data leaked into the dataset: {reservations}") for split in SPLITS: funnel[f"split_{split}_rows"] = sum(counts_by_split[split].values()) outputs = {} for split in SPLITS: path = output_directory / f"{split}.parquet" outputs[split] = { "path": str(path), "bytes": path.stat().st_size, "sha256": sha256_file(path), "rows": sum(counts_by_split[split].values()), "videos": len(videos_by_split[split]), "channels": len(channels_by_split[split]), "campaigns": len(campaigns_by_split[split]), "label_kinds": dict(counts_by_split[split]), } manifest = { "schema_version": 1, "generated_at": datetime.now(tz=UTC).isoformat(), "configuration": { "seed": seed, "train_fraction": train_fraction, "validation_fraction": validation_fraction, "test_fraction": round(1 - train_fraction - validation_fraction, 6), "encoder": encoder, "encoder_revision": encoder_revision, "max_length": max_length, "overlap_tokens": overlap_tokens, "language_prefixes": list(language_prefixes), "positive_categories": sorted(positive), "hard_negative_categories": sorted(hard_negative), "ordinary_negative_windows_per_video": ordinary_negative_windows_per_video, "maximum_video_duration_seconds": maximum_video_duration_seconds, "minimum_cues": minimum_cues, "exclude_split_directories": [str(path) for path in exclude_split_directories], }, "sources": { "subtitles_directory": str(subtitles_directory), "metadata_directory": str(metadata_directory), "mirror_path": str(mirror_path), "annotations_path": str(annotations_path) if annotations_path else None, "annotations_sha256": ( sha256_file(annotations_path) if annotations_path is not None and annotations_path.exists() else None ), "training_evidence_path": ( str(training_evidence_path) if training_evidence_path else None ), "training_evidence_sha256": ( sha256_file(training_evidence_path) if training_evidence_path is not None and training_evidence_path.exists() else None ), "provenance_path": str(provenance_path) if provenance_path else None, "provenance_sha256": ( sha256_file(provenance_path) if provenance_path is not None and provenance_path.exists() else None ), }, "exclusions": { "excluded_videos": len(excluded_videos), "excluded_channels": len(excluded_channels), }, "reservations": reservations, "funnel": dict(funnel), "label_kinds": dict(label_kind_counts), "groups": { "components": len(component_members), "accepted_videos": len(accepted_videos), "channels": len(first_by_channel), "exact_campaigns": len(first_by_campaign), "component_size_videos": { "p50": component_percentile(0.50), "p95": component_percentile(0.95), "p99": component_percentile(0.99), "max": component_sizes[-1] if component_sizes else 0, }, }, "leakage": leakage, "outputs": outputs, "elapsed_seconds": round(time.monotonic() - started, 3), } write_json_atomic(manifest_path, manifest) return manifest