Download code/src/sponsor_detection/data/transcripts.py from CuriousDragon/flow-smart-segments-20261004: direct link, hf CLI and curl.
- Browser
- Download file 17.8 kB
-
https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/data/transcripts.py
- Command line
-
hf download hf://CuriousDragon/flow-smart-segments-20261004/code/src/sponsor_detection/data/transcripts.py
-
curl -L -o transcripts.py https://huggingface.co/CuriousDragon/flow-smart-segments-20261004/resolve/main/code/src/sponsor_detection/data/transcripts.py
17.8 kB
| from __future__ import annotations | |
| import json | |
| import random | |
| import re | |
| import time | |
| from collections import Counter | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Callable, Protocol, Sequence | |
| from sponsor_detection.data.profile import sha256_file, write_json_atomic | |
| VIDEO_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]{11}$") | |
| PERMANENT_FAILURE_STATUSES = frozenset( | |
| {"authentication_required", "invalid_video_id", "no_transcript", "unavailable"} | |
| ) | |
| class TranscriptClient(Protocol): | |
| def list(self, video_id: str): ... | |
| def _load_pilot_video_ids(pilot_path: Path, expected_sha256: str) -> list[str]: | |
| if sha256_file(pilot_path) != expected_sha256: | |
| raise ValueError("pilot JSONL checksum differs from its manifest") | |
| video_ids: list[str] = [] | |
| with pilot_path.open("r", encoding="utf-8") as source: | |
| for line_number, line in enumerate(source, start=1): | |
| try: | |
| record = json.loads(line) | |
| video_ids.append(str(record["video_id"])) | |
| except (KeyError, TypeError, json.JSONDecodeError) as error: | |
| raise ValueError(f"invalid pilot record at line {line_number}") from error | |
| return video_ids | |
| def _classify_error(error: BaseException) -> tuple[str, bool]: | |
| from sponsor_detection.data.transcript_api import ( | |
| TranscriptApiConfigurationError, | |
| TranscriptApiError, | |
| TranscriptApiNoTranscriptError, | |
| TranscriptApiRetryableError, | |
| ) | |
| from sponsor_detection.data.yt_dlp_transcripts import ( | |
| YtDlpBlockedError, | |
| YtDlpNoTranscriptError, | |
| YtDlpTranscriptError, | |
| YtDlpUnavailableError, | |
| ) | |
| from youtube_transcript_api._errors import ( | |
| AgeRestricted, | |
| InvalidVideoId, | |
| IpBlocked, | |
| NoTranscriptFound, | |
| PoTokenRequired, | |
| RequestBlocked, | |
| TranscriptsDisabled, | |
| VideoUnavailable, | |
| VideoUnplayable, | |
| YouTubeTranscriptApiException, | |
| ) | |
| if isinstance(error, (RequestBlocked, IpBlocked, YtDlpBlockedError)): | |
| return "blocked", True | |
| if isinstance( | |
| error, | |
| ( | |
| TranscriptsDisabled, | |
| NoTranscriptFound, | |
| YtDlpNoTranscriptError, | |
| TranscriptApiNoTranscriptError, | |
| ), | |
| ): | |
| return "no_transcript", False | |
| if isinstance(error, TranscriptApiConfigurationError): | |
| return "provider_error", True | |
| if isinstance(error, TranscriptApiRetryableError): | |
| return "provider_unavailable", True | |
| if isinstance( | |
| error, | |
| ( | |
| AgeRestricted, | |
| InvalidVideoId, | |
| VideoUnavailable, | |
| VideoUnplayable, | |
| YtDlpUnavailableError, | |
| ), | |
| ): | |
| return "unavailable", False | |
| if isinstance(error, PoTokenRequired): | |
| return "authentication_required", False | |
| if isinstance(error, YouTubeTranscriptApiException): | |
| return "api_error", False | |
| if isinstance(error, YtDlpTranscriptError): | |
| return "api_error", False | |
| if isinstance(error, TranscriptApiError): | |
| return "api_error", False | |
| return "unexpected_error", False | |
| def _fetch_one( | |
| client: TranscriptClient, | |
| video_id: str, | |
| languages: Sequence[str], | |
| ) -> dict[str, object]: | |
| transcript = client.list(video_id).find_transcript(languages) | |
| fetched = transcript.fetch() | |
| cues = [] | |
| for index, snippet in enumerate(fetched): | |
| start_ms = round(snippet.start * 1000) | |
| duration_ms = max(0, round(snippet.duration * 1000)) | |
| cues.append( | |
| { | |
| "index": index, | |
| "start_ms": start_ms, | |
| "end_ms": start_ms + duration_ms, | |
| "text": snippet.text, | |
| } | |
| ) | |
| return { | |
| "schema_version": 1, | |
| "video_id": video_id, | |
| "source": getattr(transcript, "source", "youtube_transcript_api"), | |
| "source_version": getattr(transcript, "source_version", None), | |
| "language": transcript.language, | |
| "language_code": transcript.language_code, | |
| "is_generated": bool(transcript.is_generated), | |
| "fetched_at": datetime.now(tz=UTC).isoformat(), | |
| "cues": cues, | |
| } | |
| def acquire_transcript_canary( | |
| pilot_path: Path, | |
| pilot_manifest_path: Path, | |
| output_directory: Path, | |
| report_path: Path, | |
| *, | |
| languages: Sequence[str], | |
| limit: int, | |
| request_delay_seconds: float, | |
| stop_on_block: bool, | |
| client: TranscriptClient | None = None, | |
| sleep: Callable[[float], None] = time.sleep, | |
| progress_callback: Callable[[int, int, str], None] | None = None, | |
| ) -> dict[str, object]: | |
| if limit <= 0: | |
| raise ValueError("limit must be positive") | |
| if request_delay_seconds < 0: | |
| raise ValueError("request delay cannot be negative") | |
| if not languages: | |
| raise ValueError("at least one transcript language is required") | |
| pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) | |
| pilot_sha256 = pilot_manifest.get("output", {}).get("sha256") | |
| if not pilot_sha256: | |
| raise ValueError("pilot manifest does not contain an output SHA-256 fingerprint") | |
| video_ids = _load_pilot_video_ids(pilot_path, pilot_sha256)[:limit] | |
| if client is None: | |
| from youtube_transcript_api import YouTubeTranscriptApi | |
| client = YouTubeTranscriptApi() | |
| output_directory.mkdir(parents=True, exist_ok=True) | |
| state_path = output_directory / "state.json" | |
| state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.is_file() else {} | |
| statuses: Counter[str] = Counter() | |
| languages_found: Counter[str] = Counter() | |
| attempted = successful = total_cues = 0 | |
| stopped_for_block = False | |
| started = time.monotonic() | |
| for index, video_id in enumerate(video_ids, start=1): | |
| previous = state.get(video_id) | |
| if isinstance(previous, dict) and previous.get("status") == "success": | |
| transcript_path = output_directory / f"{video_id}.json" | |
| if transcript_path.is_file(): | |
| transcript = json.loads(transcript_path.read_text(encoding="utf-8")) | |
| statuses["cached_success"] += 1 | |
| successful += 1 | |
| total_cues += len(transcript["cues"]) | |
| languages_found[str(transcript["language_code"])] += 1 | |
| continue | |
| attempted += 1 | |
| if not VIDEO_ID_PATTERN.fullmatch(video_id): | |
| status = "invalid_video_id" | |
| state[video_id] = {"status": status, "updated_at": datetime.now(tz=UTC).isoformat()} | |
| else: | |
| try: | |
| transcript = _fetch_one(client, video_id, languages) | |
| transcript_path = output_directory / f"{video_id}.json" | |
| write_json_atomic(transcript_path, transcript) | |
| status = "success" | |
| successful += 1 | |
| total_cues += len(transcript["cues"]) | |
| languages_found[str(transcript["language_code"])] += 1 | |
| state[video_id] = { | |
| "status": status, | |
| "transcript_sha256": sha256_file(transcript_path), | |
| "updated_at": datetime.now(tz=UTC).isoformat(), | |
| } | |
| except BaseException as error: | |
| if isinstance(error, (KeyboardInterrupt, SystemExit)): | |
| raise | |
| status, should_stop = _classify_error(error) | |
| state[video_id] = { | |
| "status": status, | |
| "error_type": type(error).__name__, | |
| "updated_at": datetime.now(tz=UTC).isoformat(), | |
| } | |
| if should_stop and stop_on_block: | |
| stopped_for_block = True | |
| statuses[status] += 1 | |
| write_json_atomic(state_path, state) | |
| if progress_callback: | |
| progress_callback(index, len(video_ids), status) | |
| if stopped_for_block: | |
| break | |
| if index < len(video_ids) and request_delay_seconds: | |
| sleep(request_delay_seconds) | |
| report = { | |
| "schema_version": 1, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "source_pilot_sha256": pilot_sha256, | |
| "configuration": { | |
| "languages": list(languages), | |
| "limit": limit, | |
| "request_delay_seconds": request_delay_seconds, | |
| "stop_on_block": stop_on_block, | |
| }, | |
| "counts": { | |
| "attempted": attempted, | |
| "successful": successful, | |
| "total_cues": total_cues, | |
| "statuses": dict(statuses), | |
| "languages": dict(languages_found), | |
| }, | |
| "stopped_for_block": stopped_for_block, | |
| "elapsed_seconds": round(time.monotonic() - started, 3), | |
| } | |
| write_json_atomic(report_path, report) | |
| return report | |
| def collect_transcripts( | |
| pilot_path: Path, | |
| pilot_manifest_path: Path, | |
| output_directory: Path, | |
| report_path: Path, | |
| *, | |
| languages: Sequence[str], | |
| target_successes: int, | |
| max_attempts_per_run: int, | |
| minimum_request_delay_seconds: float, | |
| maximum_request_delay_seconds: float, | |
| max_transient_attempts_per_video: int, | |
| stop_on_block: bool, | |
| block_cooldown_seconds: float = 0, | |
| provider_name: str = "youtube_transcript_api", | |
| client: TranscriptClient | None = None, | |
| sleep: Callable[[float], None] = time.sleep, | |
| random_delay: Callable[[float, float], float] = random.uniform, | |
| progress_callback: Callable[[int, int, int, str], None] | None = None, | |
| ) -> dict[str, object]: | |
| if target_successes <= 0: | |
| raise ValueError("target_successes must be positive") | |
| if max_attempts_per_run <= 0: | |
| raise ValueError("max_attempts_per_run must be positive") | |
| if max_transient_attempts_per_video <= 0: | |
| raise ValueError("max_transient_attempts_per_video must be positive") | |
| if minimum_request_delay_seconds < 0: | |
| raise ValueError("minimum request delay cannot be negative") | |
| if maximum_request_delay_seconds < minimum_request_delay_seconds: | |
| raise ValueError("maximum request delay cannot be less than the minimum") | |
| if block_cooldown_seconds < 0: | |
| raise ValueError("block cooldown cannot be negative") | |
| if not languages: | |
| raise ValueError("at least one transcript language is required") | |
| pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) | |
| pilot_sha256 = pilot_manifest.get("output", {}).get("sha256") | |
| if not pilot_sha256: | |
| raise ValueError("pilot manifest does not contain an output SHA-256 fingerprint") | |
| video_ids = _load_pilot_video_ids(pilot_path, pilot_sha256) | |
| if client is None: | |
| from youtube_transcript_api import YouTubeTranscriptApi | |
| client = YouTubeTranscriptApi() | |
| output_directory.mkdir(parents=True, exist_ok=True) | |
| state_path = output_directory / "state.json" | |
| state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.is_file() else {} | |
| existing_successes = sum( | |
| 1 | |
| for video_id, record in state.items() | |
| if isinstance(record, dict) | |
| and record.get("status") == "success" | |
| and (output_directory / f"{video_id}.json").is_file() | |
| ) | |
| total_successes = existing_successes | |
| new_successes = attempts_this_run = 0 | |
| skipped_permanent = skipped_retry_exhausted = 0 | |
| statuses: Counter[str] = Counter() | |
| stopped_for_block = False | |
| started = time.monotonic() | |
| cooldown_remaining_seconds = _block_cooldown_remaining_seconds( | |
| state, block_cooldown_seconds | |
| ) | |
| if cooldown_remaining_seconds > 0: | |
| report = { | |
| "schema_version": 1, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "source_pilot_sha256": pilot_sha256, | |
| "configuration": { | |
| "provider": provider_name, | |
| "block_cooldown_seconds": block_cooldown_seconds, | |
| }, | |
| "counts": { | |
| "candidate_videos": len(video_ids), | |
| "existing_successes": existing_successes, | |
| "attempts_this_run": 0, | |
| "new_successes": 0, | |
| "total_successes": total_successes, | |
| "skipped_permanent": 0, | |
| "skipped_retry_exhausted": 0, | |
| "statuses_this_run": {"cooldown_active": 1}, | |
| }, | |
| "target_reached": total_successes >= target_successes, | |
| "stopped_for_block": True, | |
| "cooldown_active": True, | |
| "cooldown_remaining_seconds": cooldown_remaining_seconds, | |
| "elapsed_seconds": round(time.monotonic() - started, 3), | |
| } | |
| write_json_atomic(report_path, report) | |
| return report | |
| for video_id in video_ids: | |
| if total_successes >= target_successes or attempts_this_run >= max_attempts_per_run: | |
| break | |
| previous = state.get(video_id) | |
| previous_status = previous.get("status") if isinstance(previous, dict) else None | |
| transcript_path = output_directory / f"{video_id}.json" | |
| if previous_status == "success" and transcript_path.is_file(): | |
| continue | |
| if previous_status in PERMANENT_FAILURE_STATUSES: | |
| skipped_permanent += 1 | |
| continue | |
| prior_attempts = int(previous.get("attempts", 1)) if isinstance(previous, dict) else 0 | |
| if previous_status and prior_attempts >= max_transient_attempts_per_video: | |
| skipped_retry_exhausted += 1 | |
| continue | |
| if not VIDEO_ID_PATTERN.fullmatch(video_id): | |
| status = "invalid_video_id" | |
| state[video_id] = { | |
| "status": status, | |
| "attempts": prior_attempts, | |
| "updated_at": datetime.now(tz=UTC).isoformat(), | |
| } | |
| statuses[status] += 1 | |
| write_json_atomic(state_path, state) | |
| continue | |
| attempts_this_run += 1 | |
| try: | |
| transcript = _fetch_one(client, video_id, languages) | |
| write_json_atomic(transcript_path, transcript) | |
| status = "success" | |
| new_successes += 1 | |
| total_successes += 1 | |
| state[video_id] = { | |
| "status": status, | |
| "attempts": prior_attempts + 1, | |
| "transcript_sha256": sha256_file(transcript_path), | |
| "updated_at": datetime.now(tz=UTC).isoformat(), | |
| } | |
| except BaseException as error: | |
| if isinstance(error, (KeyboardInterrupt, SystemExit)): | |
| raise | |
| status, should_stop = _classify_error(error) | |
| state[video_id] = { | |
| "status": status, | |
| "attempts": prior_attempts + 1, | |
| "error_type": type(error).__name__, | |
| "updated_at": datetime.now(tz=UTC).isoformat(), | |
| } | |
| if should_stop and stop_on_block: | |
| stopped_for_block = True | |
| statuses[status] += 1 | |
| write_json_atomic(state_path, state) | |
| if progress_callback: | |
| progress_callback( | |
| attempts_this_run, | |
| max_attempts_per_run, | |
| total_successes, | |
| status, | |
| ) | |
| if stopped_for_block: | |
| break | |
| if attempts_this_run < max_attempts_per_run and total_successes < target_successes: | |
| delay = random_delay( | |
| minimum_request_delay_seconds, | |
| maximum_request_delay_seconds, | |
| ) | |
| if delay: | |
| sleep(delay) | |
| report = { | |
| "schema_version": 1, | |
| "generated_at": datetime.now(tz=UTC).isoformat(), | |
| "source_pilot_sha256": pilot_sha256, | |
| "configuration": { | |
| "provider": provider_name, | |
| "languages": list(languages), | |
| "target_successes": target_successes, | |
| "max_attempts_per_run": max_attempts_per_run, | |
| "minimum_request_delay_seconds": minimum_request_delay_seconds, | |
| "maximum_request_delay_seconds": maximum_request_delay_seconds, | |
| "max_transient_attempts_per_video": max_transient_attempts_per_video, | |
| "stop_on_block": stop_on_block, | |
| "block_cooldown_seconds": block_cooldown_seconds, | |
| }, | |
| "counts": { | |
| "candidate_videos": len(video_ids), | |
| "existing_successes": existing_successes, | |
| "attempts_this_run": attempts_this_run, | |
| "new_successes": new_successes, | |
| "total_successes": total_successes, | |
| "skipped_permanent": skipped_permanent, | |
| "skipped_retry_exhausted": skipped_retry_exhausted, | |
| "statuses_this_run": dict(statuses), | |
| }, | |
| "target_reached": total_successes >= target_successes, | |
| "stopped_for_block": stopped_for_block, | |
| "cooldown_active": False, | |
| "elapsed_seconds": round(time.monotonic() - started, 3), | |
| } | |
| write_json_atomic(report_path, report) | |
| return report | |
| def _block_cooldown_remaining_seconds( | |
| state: dict[str, object], block_cooldown_seconds: float | |
| ) -> int: | |
| if not block_cooldown_seconds: | |
| return 0 | |
| blocked_at: list[datetime] = [] | |
| for record in state.values(): | |
| if not isinstance(record, dict) or record.get("status") != "blocked": | |
| continue | |
| updated_at = record.get("updated_at") | |
| if not isinstance(updated_at, str): | |
| continue | |
| try: | |
| parsed = datetime.fromisoformat(updated_at) | |
| except ValueError: | |
| continue | |
| if parsed.tzinfo is not None: | |
| blocked_at.append(parsed.astimezone(UTC)) | |
| if not blocked_at: | |
| return 0 | |
| elapsed = (datetime.now(tz=UTC) - max(blocked_at)).total_seconds() | |
| return max(0, round(block_cooldown_seconds - elapsed)) | |