from __future__ import annotations
# ZeroGPU must be imported before torch so its CUDA emulation can be installed.
import os
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("OMP_NUM_THREADS", "2")
os.environ.setdefault("MKL_NUM_THREADS", "2")
import spaces
import html
import json
import logging
import math
import re
import shutil
import subprocess
import tempfile
import time
import uuid
import zipfile
from pathlib import Path
from typing import Any
import torch
import gradio as gr
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
from subtitle_utils import (
build_cues, chunks_to_words, format_duration, timed_words_to_text,
timestamp_chunk_coverage, to_json, to_srt, to_vtt, transcript_coverage,
)
APP_TITLE = "CaptionCraft AI"
MODEL_ID = "openai/whisper-large-v3"
MAX_UPLOAD_MB = int(os.getenv("MAX_UPLOAD_MB", "100"))
MAX_AUDIO_MINUTES = int(os.getenv("MAX_AUDIO_MINUTES", "20"))
OUTPUT_DIR = Path(tempfile.gettempdir()) / "captioncraft_downloads"
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
logger = logging.getLogger("captioncraft")
try:
torch.set_num_threads(max(1, min(2, int(os.getenv("CPU_THREADS", "2")))))
torch.set_num_interop_threads(1)
except RuntimeError:
# PyTorch may already have initialized its thread pools in some host images.
pass
# In a ZeroGPU Space, importing `spaces` first makes CUDA available through its
# startup emulation. On a local CPU development machine, keep a CPU fallback.
MODEL_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
MODEL_DTYPE = torch.float16 if MODEL_DEVICE == "cuda" else torch.float32
if os.getenv("SPACE_ID") and MODEL_DEVICE != "cuda":
logger.warning("No CUDA device detected. Set the Space hardware to ZeroGPU in Settings.")
logger.info("Loading %s at startup on %s (%s)", MODEL_ID, MODEL_DEVICE, MODEL_DTYPE)
processor = AutoProcessor.from_pretrained(MODEL_ID)
model = AutoModelForSpeechSeq2Seq.from_pretrained(
MODEL_ID,
dtype=MODEL_DTYPE,
low_cpu_mem_usage=True,
use_safetensors=True,
attn_implementation="sdpa",
)
model.to(MODEL_DEVICE)
model.eval()
_ASR_PIPELINE = pipeline(
task="automatic-speech-recognition",
model=model,
tokenizer=processor.tokenizer,
feature_extractor=processor.feature_extractor,
device=MODEL_DEVICE,
dtype=MODEL_DTYPE,
)
logger.info("Whisper pipeline ready on %s; CUDA available=%s", MODEL_DEVICE, torch.cuda.is_available())
LANGUAGES = [
("Auto-detect (recommended)", "auto"),
("English", "english"),
("Hindi", "hindi"),
("Bengali", "bengali"),
("Tamil", "tamil"),
("Telugu", "telugu"),
("Marathi", "marathi"),
("Gujarati", "gujarati"),
("Punjabi", "punjabi"),
("Urdu", "urdu"),
("Spanish", "spanish"),
("French", "french"),
("German", "german"),
("Portuguese", "portuguese"),
("Arabic", "arabic"),
("Japanese", "japanese"),
("Korean", "korean"),
("Chinese", "chinese"),
("Russian", "russian"),
("Indonesian", "indonesian"),
("Turkish", "turkish"),
]
def _run_command(command: list[str], timeout: int, purpose: str) -> subprocess.CompletedProcess:
try:
completed = subprocess.run(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
check=False,
)
except FileNotFoundError as exc:
raise RuntimeError("FFmpeg is not installed on this host. The Space needs the packages.txt dependency.") from exc
except subprocess.TimeoutExpired as exc:
raise RuntimeError(f"Audio {purpose} timed out. Try a shorter or smaller audio file.") from exc
if completed.returncode != 0:
detail = (completed.stderr or "").strip().splitlines()[-1:]
raise RuntimeError(f"Could not {purpose} this audio file. {detail[0] if detail else 'Unsupported or damaged audio.'}")
return completed
def _probe_duration(audio_path: Path) -> float:
command = [
"ffprobe", "-v", "error", "-select_streams", "a:0",
"-show_entries", "stream=duration:format=duration", "-of", "json", str(audio_path),
]
completed = _run_command(command, timeout=45, purpose="inspect")
try:
payload = json.loads(completed.stdout)
candidates = []
streams = payload.get("streams") or []
if streams:
candidates.append(streams[0].get("duration"))
candidates.append((payload.get("format") or {}).get("duration"))
duration = next(float(value) for value in candidates if value not in (None, "N/A", ""))
except (ValueError, TypeError, IndexError, StopIteration, json.JSONDecodeError):
raise RuntimeError("Could not read the audio duration. Please try a standard MP3, WAV, M4A, FLAC or OGG file.")
if not math.isfinite(duration) or duration <= 0:
raise RuntimeError("The uploaded file does not contain a readable audio stream.")
if duration > MAX_AUDIO_MINUTES * 60:
raise RuntimeError(f"Audio is {format_duration(duration)} long. The current limit is {MAX_AUDIO_MINUTES} minutes.")
return duration
def _prepare_audio(source: Path, target: Path, duration: float) -> None:
timeout = max(120, min(1800, int(duration * 0.75) + 90))
command = [
"ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y",
"-i", str(source), "-map", "0:a:0", "-vn", "-sn", "-dn",
"-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", str(target),
]
_run_command(command, timeout=timeout, purpose="convert")
if not target.exists() or target.stat().st_size < 1024:
raise RuntimeError("Audio conversion produced an empty file. Please check the uploaded audio.")
def _clean_old_outputs(max_age_hours: int = 8) -> None:
cutoff = time.time() - max_age_hours * 3600
try:
for child in OUTPUT_DIR.iterdir():
try:
if child.stat().st_mtime < cutoff:
if child.is_dir():
shutil.rmtree(child, ignore_errors=True)
else:
child.unlink(missing_ok=True)
except OSError:
continue
except OSError:
logger.warning("Could not clean old output files", exc_info=True)
def _safe_stem(path: Path) -> str:
stem = re.sub(r"[^A-Za-z0-9_-]+", "_", path.stem).strip("_-")
return (stem[:50] or "audio")
def _language_label(value: str) -> str:
return next((label for label, code in LANGUAGES if code == value), "Auto-detect")
def _estimate_gpu_duration(audio_file: str | None = None, *args, **kwargs) -> int:
"""Reserve a realistic amount of ZeroGPU time based on the uploaded audio length."""
if not audio_file:
return 60
try:
audio_seconds = _probe_duration(Path(audio_file))
except Exception:
# Keep the scheduler estimate conservative when metadata cannot be probed.
return 60
# Full Whisper large-v3 uses more decoder work than medium. Keep the
# reservation below the Space's 240-second cap and realistic for the audio.
return max(45, min(240, int(math.ceil(35 + audio_seconds * 0.14))))
@spaces.GPU(duration=_estimate_gpu_duration)
def generate_subtitles(
audio_file: str | None,
language: str,
task_mode: str,
max_line_chars: int,
export_formats: list[str] | None,
progress=gr.Progress(track_tqdm=False),
):
"""ZeroGPU Gradio callback: audio in, subtitle files out. No video input/output."""
work_dir: Path | None = None
try:
if os.getenv("SPACE_ID") and MODEL_DEVICE != "cuda":
raise RuntimeError(
"This Space is not running on ZeroGPU. Open Settings → Hardware and select ZeroGPU, then restart the Space."
)
if not audio_file:
raise ValueError("Upload an audio file first.")
source = Path(audio_file)
if not source.is_file():
raise ValueError("The uploaded audio file is no longer available. Please upload it again.")
size_mb = source.stat().st_size / (1024 * 1024)
if size_mb > MAX_UPLOAD_MB:
raise ValueError(f"File is {size_mb:.1f} MB. The upload limit is {MAX_UPLOAD_MB} MB.")
selected = {str(item).lower() for item in (export_formats or [])}
allowed = {"srt", "vtt", "txt", "json"}
selected &= allowed
if not selected:
raise ValueError("Select at least one subtitle format to export.")
progress(0.03, desc="Checking audio file…")
_clean_old_outputs()
duration = _probe_duration(source)
work_dir = Path(tempfile.mkdtemp(prefix="captioncraft_work_"))
wav_path = work_dir / "audio_16k_mono.wav"
progress(0.08, desc="Preparing lightweight 16 kHz mono audio…")
_prepare_audio(source, wav_path, duration)
progress(0.15, desc="Whisper-large-v3 is ready; reserving ZeroGPU for transcription…")
asr = _ASR_PIPELINE
# Accuracy-first long-form mode: do NOT set chunk_length_s. Transformers
# then uses Whisper's sequential sliding-window algorithm rather than
# independently decoding and stitching short chunks. The official
# large-v3 guide recommends this when accuracy matters more than speed.
generate_kwargs: dict[str, Any] = {
"task": "translate" if task_mode == "translate" else "transcribe",
"max_new_tokens": 448,
"num_beams": 1,
"condition_on_prev_tokens": False,
"compression_ratio_threshold": 1.35,
"temperature": (0.0, 0.2, 0.4, 0.6, 0.8, 1.0),
"logprob_threshold": -1.0,
"no_speech_threshold": 0.6,
}
if language and language != "auto":
generate_kwargs["language"] = language
progress(0.22, desc="Transcribing with accuracy-first long-form decoding…")
started = time.perf_counter()
try:
with torch.inference_mode():
result = asr(
str(wav_path),
return_timestamps="word",
generate_kwargs=generate_kwargs,
)
timestamp_quality = "word timestamps"
except (ValueError, RuntimeError) as exc:
message = str(exc).lower()
if not any(term in message for term in ("word", "timestamp", "alignment", "dtw", "dynamic time")):
raise
logger.warning("Word timestamps unavailable; retrying with segment timestamps: %s", exc)
with torch.inference_mode():
result = asr(
str(wav_path),
return_timestamps=True,
generate_kwargs=generate_kwargs,
)
timestamp_quality = "segment timestamps (word alignment unavailable)"
elapsed = time.perf_counter() - started
transcript = str(result.get("text") or "").strip()
if not transcript:
raise RuntimeError("No speech was detected. Try a clearer recording with less background noise.")
progress(0.82, desc="Checking transcript completeness and building subtitle timings…")
chunks = result.get("chunks") or []
words = chunks_to_words(chunks, transcript, duration, int(max_line_chars))
coverage = transcript_coverage(transcript, timed_words_to_text(words))
alignment_coverage = timestamp_chunk_coverage(chunks)
if not chunks or alignment_coverage < 0.999:
timestamp_quality = "word timestamps (missing spans interpolated)"
# DTW can occasionally align only part of the recognised text. Retry
# with segment timestamps if text or timestamp coverage is incomplete.
needs_alignment_repair = (
not chunks or alignment_coverage < 0.999 or coverage < 0.985
)
if needs_alignment_repair and "segment timestamps" not in timestamp_quality:
logger.warning(
"Word alignment needs repair (text=%.2f%%, timestamped chunks=%.2f%%); retrying segment timestamps",
coverage * 100, alignment_coverage * 100,
)
try:
with torch.inference_mode():
segment_result = asr(
str(wav_path),
return_timestamps=True,
generate_kwargs=generate_kwargs,
)
segment_text = str(segment_result.get("text") or "").strip()
if segment_text:
segment_words = chunks_to_words(
segment_result.get("chunks") or [], segment_text, duration, int(max_line_chars)
)
segment_coverage = transcript_coverage(segment_text, timed_words_to_text(segment_words))
if segment_coverage >= coverage:
result = segment_result
transcript = segment_text
words = segment_words
coverage = segment_coverage
alignment_coverage = timestamp_chunk_coverage(segment_result.get("chunks") or [])
timestamp_quality = "segment timestamps (completeness repair)"
except Exception:
logger.exception("Segment timestamp repair failed; retaining complete transcript with approximate timings")
# Last-resort guard: never silently discard recognised transcript text.
# When alignment is still incomplete, preserve all text and mark timings
# as approximate rather than presenting a partial subtitle file.
if coverage < 0.985 or not words or alignment_coverage == 0.0:
logger.warning("Timestamp character coverage remains %.2f%%; using full-transcript timing fallback", coverage * 100)
words = chunks_to_words([], transcript, duration, int(max_line_chars))
coverage = transcript_coverage(transcript, timed_words_to_text(words))
timestamp_quality = "full-transcript fallback (approximate timings)"
cues = build_cues(words, max_chars=int(max_line_chars))
if not cues:
raise RuntimeError("Whisper returned text but no usable timestamps. Please try another audio file.")
output_id = uuid.uuid4().hex[:10]
job_dir = OUTPUT_DIR / output_id
job_dir.mkdir(parents=True, exist_ok=True)
base_name = _safe_stem(source) + "_captions"
generated: dict[str, Path] = {}
if "srt" in selected:
generated["srt"] = job_dir / f"{base_name}.srt"
generated["srt"].write_text(to_srt(cues), encoding="utf-8")
if "vtt" in selected:
generated["vtt"] = job_dir / f"{base_name}.vtt"
generated["vtt"].write_text(to_vtt(cues), encoding="utf-8")
if "txt" in selected:
generated["txt"] = job_dir / f"{base_name}.txt"
generated["txt"].write_text(transcript + "\n", encoding="utf-8")
if "json" in selected:
generated["json"] = job_dir / f"{base_name}.json"
generated["json"].write_text(to_json(cues), encoding="utf-8")
zip_path = job_dir / f"{base_name}_all_formats.zip"
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
for fmt, file_path in generated.items():
archive.write(file_path, arcname=file_path.name)
metadata = {
"model": MODEL_ID,
"decoding": {
"max_new_tokens": 448,
"temperature_fallback": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0],
"compression_ratio_threshold": 1.35,
"logprob_threshold": -1.0,
"no_speech_threshold": 0.6,
"long_form": "accuracy-first sequential sliding-window",
},
"transcript_character_coverage": round(coverage, 5),
"timestamped_chunk_coverage": round(alignment_coverage, 5),
"execution_device": MODEL_DEVICE,
"execution_backend": "Hugging Face ZeroGPU" if MODEL_DEVICE == "cuda" else "local CPU fallback",
"task": "translate to English" if task_mode == "translate" else "transcribe",
"selected_language": _language_label(language),
"audio_duration_seconds": round(duration, 3),
"subtitle_cues": len(cues),
"timestamp_mode": timestamp_quality,
"note": "Word-level DTW alignment is approximate. Missing timestamps are repaired from nearby spans; if alignment remains incomplete, the complete transcript is preserved with approximate full-duration timings.",
}
archive.writestr("metadata.json", json.dumps(metadata, ensure_ascii=False, indent=2) + "\n")
preview = to_srt(cues)
if len(preview) > 8000:
preview = preview[:8000].rstrip() + "\n\n… Preview truncated. Download the complete file."
summary = (
f"**Completed** · {format_duration(duration)} audio · {len(cues):,} subtitle cues · "
f"{elapsed:.1f}s processing · {'ZeroGPU' if MODEL_DEVICE == 'cuda' else 'local CPU fallback'} \n"
f"Language setting: {_language_label(language)} · "
f"Mode: {'English translation' if task_mode == 'translate' else 'Original-language transcript'} · "
f"Transcript coverage guard: {coverage * 100:.1f}% · Timing: {timestamp_quality}."
)
logger.info(
"Transcribed duration=%.2fs elapsed=%.2fs cues=%d timestamp_mode=%s",
duration, elapsed, len(cues), timestamp_quality,
)
progress(1.0, desc="Subtitle files are ready.")
return (
str(generated["srt"]) if "srt" in generated else None,
str(generated["vtt"]) if "vtt" in generated else None,
str(generated["txt"]) if "txt" in generated else None,
str(generated["json"]) if "json" in generated else None,
str(zip_path),
preview,
summary,
)
except Exception as exc:
logger.exception("Subtitle generation failed")
message = str(exc).strip() or "An unexpected error occurred."
if isinstance(exc, (ValueError, RuntimeError)):
user_message = message
else:
user_message = "Subtitle generation failed. Check the audio format and try again. If this continues, check the Space logs."
return None, None, None, None, None, "", f"⚠️ **{html.escape(user_message)}**"
finally:
if work_dir:
shutil.rmtree(work_dir, ignore_errors=True)
CSS = """
:root { --cc-accent: #6d5dfc; }
.gradio-container { max-width: 1120px !important; margin: 0 auto !important; }
#cc-hero { padding: 28px 30px 24px; border: 1px solid var(--border-color-primary); border-radius: 22px; background: linear-gradient(125deg, rgba(109,93,252,.12), rgba(52,178,157,.07)); margin-bottom: 18px; }
#cc-hero h1 { font-size: clamp(28px, 4vw, 40px); line-height: 1.1; margin: 0 0 10px; letter-spacing: -1.2px; }
#cc-hero p { margin: 0; color: var(--body-text-color-subdued); font-size: 15px; }
.cc-chip { display: inline-block; margin: 0 6px 10px 0; padding: 5px 10px; border-radius: 999px; border: 1px solid var(--border-color-primary); font-size: 12px; font-weight: 600; }
#cc-panel { border: 1px solid var(--border-color-primary); border-radius: 20px; padding: 18px; }
#cc-generate { min-height: 52px; font-weight: 700; }
#cc-results { border: 1px solid var(--border-color-primary); border-radius: 18px; padding: 16px; }
.error-note { color: #c0392b; padding: 10px 12px; border-radius: 10px; background: rgba(192,57,43,.08); }
.cc-foot { text-align: center; color: var(--body-text-color-subdued); font-size: 12px; padding: 18px 8px 4px; }
@media (max-width: 700px) { #cc-hero { padding: 22px 18px; } #cc-panel { padding: 12px; } }
"""
with gr.Blocks(
title=APP_TITLE,
analytics_enabled=False,
) as demo:
gr.HTML(
"""
Turn spoken audio into clean, timed subtitle files. ZeroGPU is allocated only for transcription. Download SRT, VTT, TXT or JSON — no video rendering or burned-in captions.CaptionCraft AI