jayaspjacob Claude Opus 4.8 (1M context) commited on
Commit ·
623fe6a
1
Parent(s): 842e5ab
Add Podify AI podcast generator app
Browse filesTwo-phase Gradio app:
- Phase 1: LangGraph research agents (HF Inference LLM + DuckDuckGo) write a script
- Phase 2: Fish Audio OpenAudio S1-mini TTS on ZeroGPU with presets + voice cloning
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
- .gitattributes +3 -35
- .gitignore +4 -0
- README.md +64 -8
- app.py +199 -0
- packages.txt +2 -0
- requirements.txt +20 -0
- research/__init__.py +1 -0
- research/graph.py +210 -0
- research/llm.py +54 -0
- research/search.py +54 -0
- tts/__init__.py +1 -0
- tts/engine.py +195 -0
- tts/voices.py +63 -0
- tts/voices/README.md +10 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.wav filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.flac filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.venv/
|
| 4 |
+
*.wav.tmp
|
README.md
CHANGED
|
@@ -1,14 +1,70 @@
|
|
| 1 |
---
|
| 2 |
-
title: Podify
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
|
|
|
|
|
|
|
|
|
| 10 |
pinned: false
|
| 11 |
-
short_description:
|
| 12 |
---
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Podify - AI Podcast Generator
|
| 3 |
+
emoji: 🎙️
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: purple
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 4.44.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
python_version: "3.10"
|
| 10 |
+
hardware: zero-a10g
|
| 11 |
+
suggested_hardware: zero-a10g
|
| 12 |
pinned: false
|
| 13 |
+
short_description: Research a topic and turn it into a voiced podcast
|
| 14 |
---
|
| 15 |
|
| 16 |
+
# 🎙️ Podify — AI Podcast Generator
|
| 17 |
+
|
| 18 |
+
Turn any topic into a finished, voiced podcast in two phases:
|
| 19 |
+
|
| 20 |
+
1. **Content** — research agents (LangGraph) use a HuggingFace-hosted LLM plus live
|
| 21 |
+
DuckDuckGo web search to research the topic and write a speaker-tagged script.
|
| 22 |
+
2. **Audio** — the self-hosted **Fish Audio / OpenAudio S1-mini** model speaks the
|
| 23 |
+
script, with selectable preset voices and zero-shot **voice cloning** from an
|
| 24 |
+
uploaded clip or a live mic recording.
|
| 25 |
+
|
| 26 |
+
Everything runs inside this single Gradio Space; the TTS model runs on **ZeroGPU**.
|
| 27 |
+
|
| 28 |
+
## Architecture
|
| 29 |
+
|
| 30 |
+
```
|
| 31 |
+
Topic ─▶ LangGraph: plan ─▶ DDG search ─▶ outline ─▶ write ─▶ Script
|
| 32 |
+
Script ─▶ Fish Audio S1-mini (@spaces.GPU): per-line synth ─▶ stitched podcast WAV
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
- `app.py` — Gradio Blocks UI (two tabs) wiring both phases.
|
| 36 |
+
- `research/` — `llm.py` (HF Inference client), `search.py` (DuckDuckGo), `graph.py`
|
| 37 |
+
(LangGraph research graph).
|
| 38 |
+
- `tts/` — `engine.py` (model load + GPU synthesis + multi-speaker stitching),
|
| 39 |
+
`voices.py` (preset voice registry).
|
| 40 |
+
|
| 41 |
+
## Configuration
|
| 42 |
+
|
| 43 |
+
Set these as **Space secrets / variables** (Settings → Variables and secrets):
|
| 44 |
+
|
| 45 |
+
| Name | Required | Purpose |
|
| 46 |
+
|-------------|----------|----------------------------------------------------------------|
|
| 47 |
+
| `HF_TOKEN` | ✅ | LLM inference (Inference Providers) + model download. |
|
| 48 |
+
| `LLM_MODEL` | optional | Override the content LLM (default `meta-llama/Llama-3.3-70B-Instruct`). |
|
| 49 |
+
| `TTS_MODEL_REPO` | optional | Override the TTS model repo (default `fishaudio/openaudio-s1-mini`). |
|
| 50 |
+
|
| 51 |
+
**ZeroGPU** requires a HuggingFace **PRO** account on the Space owner.
|
| 52 |
+
|
| 53 |
+
## Run locally
|
| 54 |
+
|
| 55 |
+
```bash
|
| 56 |
+
pip install -r requirements.txt
|
| 57 |
+
export HF_TOKEN=hf_xxx # PowerShell: $env:HF_TOKEN="hf_xxx"
|
| 58 |
+
python app.py
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Phase 1 (research + script) runs on CPU. Phase 2 (TTS) needs a GPU and the
|
| 62 |
+
`fish-speech` package; on CPU-only machines the UI loads but synthesis is disabled.
|
| 63 |
+
|
| 64 |
+
## Deploy to a Space
|
| 65 |
+
|
| 66 |
+
```bash
|
| 67 |
+
huggingface-cli login
|
| 68 |
+
huggingface-cli upload <user>/podify . --repo-type=space
|
| 69 |
+
# or: git push to the Space remote (preset .wav files tracked via Git LFS)
|
| 70 |
+
```
|
app.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Podify — AI Podcast Generator (Gradio + HuggingFace Spaces).
|
| 2 |
+
|
| 3 |
+
Phase 1: research agents (LangGraph + HF LLM + DuckDuckGo) write a script.
|
| 4 |
+
Phase 2: Fish Audio (OpenAudio S1-mini) voices the script, with presets + cloning.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import List, Tuple
|
| 10 |
+
|
| 11 |
+
import gradio as gr
|
| 12 |
+
|
| 13 |
+
from research.graph import generate_script, parse_script
|
| 14 |
+
from tts import voices as voice_lib
|
| 15 |
+
from tts import engine as tts_engine
|
| 16 |
+
|
| 17 |
+
MAX_SPEAKERS = 4
|
| 18 |
+
STYLES = ["conversational", "interview", "storytelling", "educational", "debate", "news"]
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
# ------------------------------------------------------------------- Phase 1 handler
|
| 22 |
+
def run_research(topic, style, duration, num_speakers, progress=gr.Progress()):
|
| 23 |
+
topic = (topic or "").strip()
|
| 24 |
+
if not topic:
|
| 25 |
+
raise gr.Error("Please enter a topic for the podcast.")
|
| 26 |
+
progress(0.1, desc="Planning research…")
|
| 27 |
+
state = generate_script(
|
| 28 |
+
topic,
|
| 29 |
+
style=style,
|
| 30 |
+
duration_min=int(duration),
|
| 31 |
+
num_speakers=int(num_speakers),
|
| 32 |
+
)
|
| 33 |
+
progress(0.9, desc="Finalizing script…")
|
| 34 |
+
script = state.get("script", "")
|
| 35 |
+
sources = state.get("sources", [])
|
| 36 |
+
sources_md = (
|
| 37 |
+
"### Sources\n" + "\n".join(f"- {u}" for u in sources)
|
| 38 |
+
if sources
|
| 39 |
+
else "_No web sources were retrieved (the script was written from model knowledge)._"
|
| 40 |
+
)
|
| 41 |
+
return script, script, sources_md
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# ------------------------------------------------------- set up per-speaker voice rows
|
| 45 |
+
def setup_voices(script_text):
|
| 46 |
+
lines = parse_script(script_text or "")
|
| 47 |
+
# unique speakers in order of appearance
|
| 48 |
+
speakers: List[str] = []
|
| 49 |
+
for spk, _ in lines:
|
| 50 |
+
if spk not in speakers:
|
| 51 |
+
speakers.append(spk)
|
| 52 |
+
speakers = speakers[:MAX_SPEAKERS]
|
| 53 |
+
|
| 54 |
+
updates = [lines, speakers] # lines_state, speakers_state
|
| 55 |
+
presets = voice_lib.list_presets()
|
| 56 |
+
for i in range(MAX_SPEAKERS):
|
| 57 |
+
if i < len(speakers):
|
| 58 |
+
# row visibility, speaker label (markdown), preset dropdown
|
| 59 |
+
updates.append(gr.update(visible=True))
|
| 60 |
+
updates.append(gr.update(value=f"### 🎤 Voice for “{speakers[i]}”"))
|
| 61 |
+
default_choice = presets[min(i + 1, len(presets) - 1)] if len(presets) > 1 else presets[0]
|
| 62 |
+
updates.append(gr.update(choices=presets, value=default_choice))
|
| 63 |
+
else:
|
| 64 |
+
updates.append(gr.update(visible=False))
|
| 65 |
+
updates.append(gr.update())
|
| 66 |
+
updates.append(gr.update())
|
| 67 |
+
return updates
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# ------------------------------------------------------------------- Phase 2 handler
|
| 71 |
+
def run_tts(lines, speakers, *voice_inputs, progress=gr.Progress()):
|
| 72 |
+
if not tts_engine.is_available():
|
| 73 |
+
raise gr.Error(
|
| 74 |
+
"The TTS stack (fish-speech + GPU) is not available in this environment. "
|
| 75 |
+
"Deploy to a ZeroGPU Space to generate audio."
|
| 76 |
+
)
|
| 77 |
+
if not lines:
|
| 78 |
+
raise gr.Error("No script is loaded. Generate or paste a script, then click "
|
| 79 |
+
"“Prepare voices”.")
|
| 80 |
+
|
| 81 |
+
# voice_inputs is flattened: [preset0..N, clone0..N, text0..N]
|
| 82 |
+
presets = list(voice_inputs[0:MAX_SPEAKERS])
|
| 83 |
+
clones = list(voice_inputs[MAX_SPEAKERS:2 * MAX_SPEAKERS])
|
| 84 |
+
texts = list(voice_inputs[2 * MAX_SPEAKERS:3 * MAX_SPEAKERS])
|
| 85 |
+
|
| 86 |
+
voice_map = {}
|
| 87 |
+
for i, speaker in enumerate(speakers):
|
| 88 |
+
clone_path = clones[i]
|
| 89 |
+
if clone_path: # cloning overrides preset
|
| 90 |
+
voice_map[speaker] = tts_engine.VoiceConfig(
|
| 91 |
+
ref_audio=clone_path, ref_text=(texts[i] or "").strip()
|
| 92 |
+
)
|
| 93 |
+
else:
|
| 94 |
+
v = voice_lib.get_voice(presets[i])
|
| 95 |
+
voice_map[speaker] = tts_engine.VoiceConfig(ref_audio=v.ref_audio, ref_text=v.ref_text)
|
| 96 |
+
|
| 97 |
+
sr, audio = tts_engine.generate_podcast(lines, voice_map, progress=progress)
|
| 98 |
+
return (sr, audio), tts_engine.write_wav(sr, audio)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ----------------------------------------------------------------------------- UI
|
| 102 |
+
def build_ui():
|
| 103 |
+
with gr.Blocks(title="Podify — AI Podcast Generator", theme=gr.themes.Soft()) as demo:
|
| 104 |
+
gr.Markdown(
|
| 105 |
+
"# 🎙️ Podify — AI Podcast Generator\n"
|
| 106 |
+
"**Phase 1** researches your topic and writes a script. "
|
| 107 |
+
"**Phase 2** voices it with Fish Audio, with presets or your own cloned voice."
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
lines_state = gr.State([])
|
| 111 |
+
speakers_state = gr.State([])
|
| 112 |
+
|
| 113 |
+
with gr.Tab("1 · Script"):
|
| 114 |
+
with gr.Row():
|
| 115 |
+
with gr.Column(scale=2):
|
| 116 |
+
topic = gr.Textbox(
|
| 117 |
+
label="Podcast topic",
|
| 118 |
+
placeholder="e.g. How CRISPR is changing medicine",
|
| 119 |
+
lines=2,
|
| 120 |
+
)
|
| 121 |
+
with gr.Row():
|
| 122 |
+
style = gr.Dropdown(STYLES, value="conversational", label="Style")
|
| 123 |
+
duration = gr.Slider(1, 20, value=5, step=1, label="Length (min)")
|
| 124 |
+
num_speakers = gr.Slider(1, MAX_SPEAKERS, value=2, step=1, label="Speakers")
|
| 125 |
+
research_btn = gr.Button("🔎 Research & Write Script", variant="primary")
|
| 126 |
+
with gr.Column(scale=1):
|
| 127 |
+
sources_md = gr.Markdown("Sources will appear here after research.")
|
| 128 |
+
script_box = gr.Textbox(
|
| 129 |
+
label="Generated script (editable — use `Speaker: text` per line)",
|
| 130 |
+
lines=16,
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
with gr.Tab("2 · Audio"):
|
| 134 |
+
gr.Markdown(
|
| 135 |
+
"Edit the script if needed, click **Prepare voices**, assign a preset or "
|
| 136 |
+
"clone a voice per speaker (upload a clip or record from mic), then generate."
|
| 137 |
+
)
|
| 138 |
+
script_box2 = gr.Textbox(label="Script to voice", lines=12)
|
| 139 |
+
prepare_btn = gr.Button("🎚️ Prepare voices from script")
|
| 140 |
+
|
| 141 |
+
preset_dropdowns, clone_audios, clone_texts = [], [], []
|
| 142 |
+
speaker_rows, speaker_labels = [], []
|
| 143 |
+
for i in range(MAX_SPEAKERS):
|
| 144 |
+
with gr.Group(visible=False) as row:
|
| 145 |
+
speaker_label = gr.Markdown(f"### Speaker {i + 1}")
|
| 146 |
+
with gr.Row():
|
| 147 |
+
preset = gr.Dropdown(
|
| 148 |
+
voice_lib.list_presets(),
|
| 149 |
+
value=voice_lib.list_presets()[0],
|
| 150 |
+
label="Preset voice",
|
| 151 |
+
)
|
| 152 |
+
clone = gr.Audio(
|
| 153 |
+
sources=["upload", "microphone"],
|
| 154 |
+
type="filepath",
|
| 155 |
+
label="…or clone a voice (10–30s reference)",
|
| 156 |
+
)
|
| 157 |
+
ref_text = gr.Textbox(
|
| 158 |
+
label="Reference transcript (optional, improves cloning)",
|
| 159 |
+
lines=1,
|
| 160 |
+
)
|
| 161 |
+
speaker_rows.append(row)
|
| 162 |
+
speaker_labels.append(speaker_label)
|
| 163 |
+
preset_dropdowns.append(preset)
|
| 164 |
+
clone_audios.append(clone)
|
| 165 |
+
clone_texts.append(ref_text)
|
| 166 |
+
|
| 167 |
+
generate_btn = gr.Button("🎧 Generate Podcast", variant="primary")
|
| 168 |
+
audio_out = gr.Audio(label="Podcast", type="numpy")
|
| 169 |
+
file_out = gr.File(label="Download (.wav)")
|
| 170 |
+
|
| 171 |
+
# ---- wiring ----
|
| 172 |
+
# Phase 1: research -> fill both script boxes + sources, then set up voice rows.
|
| 173 |
+
setup_outputs = [lines_state, speakers_state]
|
| 174 |
+
for i in range(MAX_SPEAKERS):
|
| 175 |
+
setup_outputs += [speaker_rows[i], speaker_labels[i], preset_dropdowns[i]]
|
| 176 |
+
|
| 177 |
+
research_btn.click(
|
| 178 |
+
run_research,
|
| 179 |
+
inputs=[topic, style, duration, num_speakers],
|
| 180 |
+
outputs=[script_box, script_box2, sources_md],
|
| 181 |
+
).then(
|
| 182 |
+
setup_voices,
|
| 183 |
+
inputs=[script_box2],
|
| 184 |
+
outputs=setup_outputs,
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
prepare_btn.click(setup_voices, inputs=[script_box2], outputs=setup_outputs)
|
| 188 |
+
|
| 189 |
+
generate_btn.click(
|
| 190 |
+
run_tts,
|
| 191 |
+
inputs=[lines_state, speakers_state] + preset_dropdowns + clone_audios + clone_texts,
|
| 192 |
+
outputs=[audio_out, file_out],
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
return demo
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
if __name__ == "__main__":
|
| 199 |
+
build_ui().launch()
|
packages.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
ffmpeg
|
| 2 |
+
libsndfile1
|
requirements.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# --- Core app ---
|
| 2 |
+
# Keep gradio pinned to the README `sdk_version` so the Space runtime and pip agree.
|
| 3 |
+
gradio==4.44.0
|
| 4 |
+
spaces>=0.30.0
|
| 5 |
+
|
| 6 |
+
# --- Phase 1: research / content generation ---
|
| 7 |
+
langgraph>=0.2.0
|
| 8 |
+
langchain-core>=0.3.0
|
| 9 |
+
huggingface_hub>=0.25.0
|
| 10 |
+
ddgs>=6.0.0
|
| 11 |
+
|
| 12 |
+
# --- Phase 2: Fish Audio (OpenAudio S1-mini) TTS ---
|
| 13 |
+
# fish-speech provides the inference engine + model code. It is installed from the
|
| 14 |
+
# upstream repo (no stable PyPI release). Pin to a known-good commit when deploying.
|
| 15 |
+
fish-speech @ git+https://github.com/fishaudio/fish-speech.git
|
| 16 |
+
|
| 17 |
+
torch>=2.4.0
|
| 18 |
+
torchaudio>=2.4.0
|
| 19 |
+
numpy>=1.26.0
|
| 20 |
+
soundfile>=0.12.1
|
research/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Phase 1: research + podcast-script generation."""
|
research/graph.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""LangGraph research graph: plan -> research -> outline -> write.
|
| 2 |
+
|
| 3 |
+
Produces a speaker-tagged podcast script from a topic, grounded in live DuckDuckGo
|
| 4 |
+
search results. Kept deliberately linear and lightweight so it runs fast on a CPU Space.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import re
|
| 11 |
+
from typing import List, Tuple, TypedDict
|
| 12 |
+
|
| 13 |
+
from langgraph.graph import StateGraph, START, END
|
| 14 |
+
|
| 15 |
+
from .llm import complete
|
| 16 |
+
from .search import web_search, SearchResult
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class ResearchState(TypedDict, total=False):
|
| 20 |
+
topic: str
|
| 21 |
+
style: str
|
| 22 |
+
duration_min: int
|
| 23 |
+
num_speakers: int
|
| 24 |
+
speaker_names: List[str]
|
| 25 |
+
queries: List[str]
|
| 26 |
+
findings: str
|
| 27 |
+
sources: List[str]
|
| 28 |
+
outline: str
|
| 29 |
+
script: str
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
# --------------------------------------------------------------------------- nodes
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def plan_node(state: ResearchState) -> dict:
|
| 36 |
+
topic = state["topic"]
|
| 37 |
+
raw = complete(
|
| 38 |
+
system=(
|
| 39 |
+
"You are a research planner. Given a podcast topic, produce 3-6 focused web "
|
| 40 |
+
"search queries that together cover the key angles. Respond ONLY with a JSON "
|
| 41 |
+
'array of strings, e.g. ["query one", "query two"].'
|
| 42 |
+
),
|
| 43 |
+
user=f"Topic: {topic}",
|
| 44 |
+
temperature=0.4,
|
| 45 |
+
max_tokens=400,
|
| 46 |
+
)
|
| 47 |
+
queries = _parse_json_list(raw) or [topic]
|
| 48 |
+
return {"queries": queries[:6]}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def research_node(state: ResearchState) -> dict:
|
| 52 |
+
blocks: List[str] = []
|
| 53 |
+
sources: List[str] = []
|
| 54 |
+
for q in state.get("queries", []):
|
| 55 |
+
results: List[SearchResult] = web_search(q, max_results=4)
|
| 56 |
+
if not results:
|
| 57 |
+
continue
|
| 58 |
+
blocks.append(f"### Query: {q}\n" + "\n".join(r.as_markdown() for r in results))
|
| 59 |
+
sources.extend(r.url for r in results)
|
| 60 |
+
findings = "\n\n".join(blocks) if blocks else "(No web results were available.)"
|
| 61 |
+
# De-duplicate sources, preserve order.
|
| 62 |
+
seen, uniq = set(), []
|
| 63 |
+
for u in sources:
|
| 64 |
+
if u not in seen:
|
| 65 |
+
seen.add(u)
|
| 66 |
+
uniq.append(u)
|
| 67 |
+
return {"findings": findings, "sources": uniq}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def outline_node(state: ResearchState) -> dict:
|
| 71 |
+
outline = complete(
|
| 72 |
+
system=(
|
| 73 |
+
"You are a podcast producer. Using the research findings, write a tight "
|
| 74 |
+
"outline (intro, 3-5 segments, outro) for the podcast. Use markdown bullets."
|
| 75 |
+
),
|
| 76 |
+
user=(
|
| 77 |
+
f"Topic: {state['topic']}\n"
|
| 78 |
+
f"Style: {state.get('style', 'conversational')}\n"
|
| 79 |
+
f"Target length: ~{state.get('duration_min', 5)} minutes\n\n"
|
| 80 |
+
f"Research findings:\n{state.get('findings', '')}"
|
| 81 |
+
),
|
| 82 |
+
temperature=0.6,
|
| 83 |
+
max_tokens=800,
|
| 84 |
+
)
|
| 85 |
+
return {"outline": outline}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def write_node(state: ResearchState) -> dict:
|
| 89 |
+
speakers = state.get("speaker_names") or _default_speakers(state.get("num_speakers", 2))
|
| 90 |
+
speaker_list = ", ".join(speakers)
|
| 91 |
+
fmt = "\n".join(f"{s}: <what they say>" for s in speakers)
|
| 92 |
+
script = complete(
|
| 93 |
+
system=(
|
| 94 |
+
"You are a professional podcast scriptwriter. Write a natural, engaging, "
|
| 95 |
+
"factually-grounded podcast script based on the outline and findings.\n"
|
| 96 |
+
f"Speakers: {speaker_list}.\n"
|
| 97 |
+
"Format STRICTLY as one line per turn, prefixed with the speaker name and a "
|
| 98 |
+
f"colon, like:\n{fmt}\n"
|
| 99 |
+
"Do not include stage directions, markdown, or headings - only spoken dialogue "
|
| 100 |
+
"lines. Keep each line to a few sentences. Open with a hook and close with a "
|
| 101 |
+
"sign-off."
|
| 102 |
+
),
|
| 103 |
+
user=(
|
| 104 |
+
f"Topic: {state['topic']}\n"
|
| 105 |
+
f"Style: {state.get('style', 'conversational')}\n"
|
| 106 |
+
f"Target length: ~{state.get('duration_min', 5)} minutes\n\n"
|
| 107 |
+
f"Outline:\n{state.get('outline', '')}\n\n"
|
| 108 |
+
f"Findings:\n{state.get('findings', '')}"
|
| 109 |
+
),
|
| 110 |
+
temperature=0.8,
|
| 111 |
+
max_tokens=3000,
|
| 112 |
+
)
|
| 113 |
+
return {"script": script.strip()}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
# --------------------------------------------------------------------------- helpers
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _parse_json_list(text: str) -> List[str]:
|
| 120 |
+
match = re.search(r"\[.*\]", text, re.DOTALL)
|
| 121 |
+
if not match:
|
| 122 |
+
return [line.strip("-* ").strip() for line in text.splitlines() if line.strip()]
|
| 123 |
+
try:
|
| 124 |
+
data = json.loads(match.group(0))
|
| 125 |
+
return [str(x).strip() for x in data if str(x).strip()]
|
| 126 |
+
except json.JSONDecodeError:
|
| 127 |
+
return []
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _default_speakers(n: int) -> List[str]:
|
| 131 |
+
names = ["Host", "Guest", "Co-host", "Expert"]
|
| 132 |
+
if n <= 1:
|
| 133 |
+
return ["Narrator"]
|
| 134 |
+
return names[:n]
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def parse_script(script: str) -> List[Tuple[str, str]]:
|
| 138 |
+
"""Turn a 'Speaker: text' transcript into [(speaker, text), ...]."""
|
| 139 |
+
lines: List[Tuple[str, str]] = []
|
| 140 |
+
pattern = re.compile(r"^\s*([\w .'-]{1,30}?)\s*:\s*(.+)$")
|
| 141 |
+
for raw in script.splitlines():
|
| 142 |
+
raw = raw.strip()
|
| 143 |
+
if not raw:
|
| 144 |
+
continue
|
| 145 |
+
m = pattern.match(raw)
|
| 146 |
+
if m:
|
| 147 |
+
lines.append((m.group(1).strip(), m.group(2).strip()))
|
| 148 |
+
elif lines: # continuation of previous speaker's line
|
| 149 |
+
spk, txt = lines[-1]
|
| 150 |
+
lines[-1] = (spk, f"{txt} {raw}")
|
| 151 |
+
return lines
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
# --------------------------------------------------------------------------- graph
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def build_graph():
|
| 158 |
+
g = StateGraph(ResearchState)
|
| 159 |
+
g.add_node("plan", plan_node)
|
| 160 |
+
g.add_node("research", research_node)
|
| 161 |
+
g.add_node("outline", outline_node)
|
| 162 |
+
g.add_node("write", write_node)
|
| 163 |
+
g.add_edge(START, "plan")
|
| 164 |
+
g.add_edge("plan", "research")
|
| 165 |
+
g.add_edge("research", "outline")
|
| 166 |
+
g.add_edge("outline", "write")
|
| 167 |
+
g.add_edge("write", END)
|
| 168 |
+
return g.compile()
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
_GRAPH = None
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def generate_script(
|
| 175 |
+
topic: str,
|
| 176 |
+
*,
|
| 177 |
+
style: str = "conversational",
|
| 178 |
+
duration_min: int = 5,
|
| 179 |
+
num_speakers: int = 2,
|
| 180 |
+
speaker_names: List[str] | None = None,
|
| 181 |
+
) -> dict:
|
| 182 |
+
"""Run the full research graph and return the final state."""
|
| 183 |
+
global _GRAPH
|
| 184 |
+
if _GRAPH is None:
|
| 185 |
+
_GRAPH = build_graph()
|
| 186 |
+
speakers = speaker_names or _default_speakers(num_speakers)
|
| 187 |
+
result = _GRAPH.invoke(
|
| 188 |
+
{
|
| 189 |
+
"topic": topic,
|
| 190 |
+
"style": style,
|
| 191 |
+
"duration_min": duration_min,
|
| 192 |
+
"num_speakers": num_speakers,
|
| 193 |
+
"speaker_names": speakers,
|
| 194 |
+
}
|
| 195 |
+
)
|
| 196 |
+
return result
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
if __name__ == "__main__": # manual smoke test (needs HF_TOKEN)
|
| 200 |
+
import sys
|
| 201 |
+
|
| 202 |
+
t = sys.argv[1] if len(sys.argv) > 1 else "The history and future of electric cars"
|
| 203 |
+
out = generate_script(t, duration_min=3)
|
| 204 |
+
print("\n=== SCRIPT ===\n")
|
| 205 |
+
print(out["script"])
|
| 206 |
+
print("\n=== SOURCES ===\n")
|
| 207 |
+
print("\n".join(out.get("sources", [])))
|
| 208 |
+
print("\n=== PARSED LINES ===\n")
|
| 209 |
+
for spk, txt in parse_script(out["script"]):
|
| 210 |
+
print(f"[{spk}] {txt[:80]}")
|
research/llm.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Thin wrapper around HuggingFace Inference Providers (chat completions).
|
| 2 |
+
|
| 3 |
+
Reads ``HF_TOKEN`` from the environment and exposes a single ``chat()`` helper used by
|
| 4 |
+
the research graph. The model id is configurable via ``LLM_MODEL``.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
from functools import lru_cache
|
| 11 |
+
from typing import List, Dict
|
| 12 |
+
|
| 13 |
+
from huggingface_hub import InferenceClient
|
| 14 |
+
|
| 15 |
+
DEFAULT_MODEL = os.environ.get("LLM_MODEL", "meta-llama/Llama-3.3-70B-Instruct")
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@lru_cache(maxsize=1)
|
| 19 |
+
def _client() -> InferenceClient:
|
| 20 |
+
token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
| 21 |
+
if not token:
|
| 22 |
+
raise RuntimeError(
|
| 23 |
+
"HF_TOKEN is not set. Add it as a Space secret (or export it locally) so the "
|
| 24 |
+
"research agents can call HuggingFace Inference Providers."
|
| 25 |
+
)
|
| 26 |
+
return InferenceClient(token=token)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def chat(
|
| 30 |
+
messages: List[Dict[str, str]],
|
| 31 |
+
*,
|
| 32 |
+
model: str | None = None,
|
| 33 |
+
temperature: float = 0.7,
|
| 34 |
+
max_tokens: int = 2048,
|
| 35 |
+
) -> str:
|
| 36 |
+
"""Run a chat completion and return the assistant text."""
|
| 37 |
+
resp = _client().chat_completion(
|
| 38 |
+
messages=messages,
|
| 39 |
+
model=model or DEFAULT_MODEL,
|
| 40 |
+
temperature=temperature,
|
| 41 |
+
max_tokens=max_tokens,
|
| 42 |
+
)
|
| 43 |
+
return resp.choices[0].message.content or ""
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def complete(system: str, user: str, **kwargs) -> str:
|
| 47 |
+
"""Convenience for a single system+user turn."""
|
| 48 |
+
return chat(
|
| 49 |
+
[
|
| 50 |
+
{"role": "system", "content": system},
|
| 51 |
+
{"role": "user", "content": user},
|
| 52 |
+
],
|
| 53 |
+
**kwargs,
|
| 54 |
+
)
|
research/search.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DuckDuckGo web search tool for the research agents (free, no API key).
|
| 2 |
+
|
| 3 |
+
Uses the maintained ``ddgs`` package. Degrades gracefully on rate limits / empty
|
| 4 |
+
results so the writing step can still proceed from whatever was gathered.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import time
|
| 10 |
+
from dataclasses import dataclass
|
| 11 |
+
from typing import List
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass
|
| 15 |
+
class SearchResult:
|
| 16 |
+
title: str
|
| 17 |
+
url: str
|
| 18 |
+
snippet: str
|
| 19 |
+
|
| 20 |
+
def as_markdown(self) -> str:
|
| 21 |
+
return f"- [{self.title}]({self.url})\n {self.snippet}"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def web_search(query: str, max_results: int = 5, *, retries: int = 2) -> List[SearchResult]:
|
| 25 |
+
"""Return up to ``max_results`` results for ``query``. Never raises."""
|
| 26 |
+
try:
|
| 27 |
+
from ddgs import DDGS
|
| 28 |
+
except ImportError: # pragma: no cover - older package name fallback
|
| 29 |
+
try:
|
| 30 |
+
from duckduckgo_search import DDGS # type: ignore
|
| 31 |
+
except ImportError:
|
| 32 |
+
return []
|
| 33 |
+
|
| 34 |
+
last_err: Exception | None = None
|
| 35 |
+
for attempt in range(retries + 1):
|
| 36 |
+
try:
|
| 37 |
+
with DDGS() as ddgs:
|
| 38 |
+
hits = ddgs.text(query, max_results=max_results)
|
| 39 |
+
results = [
|
| 40 |
+
SearchResult(
|
| 41 |
+
title=h.get("title", "").strip(),
|
| 42 |
+
url=h.get("href") or h.get("url", ""),
|
| 43 |
+
snippet=(h.get("body") or h.get("snippet") or "").strip(),
|
| 44 |
+
)
|
| 45 |
+
for h in (hits or [])
|
| 46 |
+
]
|
| 47 |
+
return [r for r in results if r.url]
|
| 48 |
+
except Exception as exc: # rate limit / network hiccup
|
| 49 |
+
last_err = exc
|
| 50 |
+
if attempt < retries:
|
| 51 |
+
time.sleep(1.5 * (attempt + 1))
|
| 52 |
+
# Give up quietly; caller handles empty findings.
|
| 53 |
+
print(f"[search] '{query}' failed after {retries + 1} attempts: {last_err}")
|
| 54 |
+
return []
|
tts/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Phase 2: Fish Audio (OpenAudio S1-mini) text-to-speech + voice cloning."""
|
tts/engine.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fish Audio (OpenAudio S1-mini) inference engine.
|
| 2 |
+
|
| 3 |
+
Loads the model once (cached), exposes a ZeroGPU-decorated ``synthesize()`` for a single
|
| 4 |
+
utterance with optional zero-shot voice cloning, and ``generate_podcast()`` to stitch a
|
| 5 |
+
multi-speaker script into one waveform.
|
| 6 |
+
|
| 7 |
+
Heavy deps (torch / fish_speech) are imported lazily so this module can be imported on a
|
| 8 |
+
CPU-only / local machine (Phase 1 development) without them installed.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import os
|
| 14 |
+
import tempfile
|
| 15 |
+
from dataclasses import dataclass
|
| 16 |
+
from typing import List, Optional, Tuple
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
|
| 20 |
+
# ----------------------------------------------------------------- ZeroGPU decorator
|
| 21 |
+
try:
|
| 22 |
+
import spaces # provided in HF Spaces runtime
|
| 23 |
+
|
| 24 |
+
GPU = spaces.GPU
|
| 25 |
+
except Exception: # local / non-Space: no-op decorator
|
| 26 |
+
|
| 27 |
+
def GPU(*dargs, **dkwargs):
|
| 28 |
+
def _wrap(fn):
|
| 29 |
+
return fn
|
| 30 |
+
|
| 31 |
+
# support both @GPU and @GPU(duration=...)
|
| 32 |
+
if len(dargs) == 1 and callable(dargs[0]) and not dkwargs:
|
| 33 |
+
return dargs[0]
|
| 34 |
+
return _wrap
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
TTS_MODEL_REPO = os.environ.get("TTS_MODEL_REPO", "fishaudio/openaudio-s1-mini")
|
| 38 |
+
# Filenames inside the model repo — verify against the repo if it changes.
|
| 39 |
+
DECODER_CHECKPOINT = os.environ.get("TTS_DECODER_CKPT", "codec.pth")
|
| 40 |
+
DECODER_CONFIG = os.environ.get("TTS_DECODER_CONFIG", "modded_dac_vq")
|
| 41 |
+
|
| 42 |
+
_ENGINE = None # cached TTSInferenceEngine
|
| 43 |
+
_SAMPLE_RATE = 44100
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@dataclass
|
| 47 |
+
class VoiceConfig:
|
| 48 |
+
"""Resolved voice for one speaker: a reference clip+text, or model default."""
|
| 49 |
+
|
| 50 |
+
ref_audio: Optional[str] = None
|
| 51 |
+
ref_text: str = ""
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def is_available() -> bool:
|
| 55 |
+
"""True if the TTS stack can run (fish_speech + torch importable)."""
|
| 56 |
+
try:
|
| 57 |
+
import torch # noqa: F401
|
| 58 |
+
import fish_speech # noqa: F401
|
| 59 |
+
|
| 60 |
+
return True
|
| 61 |
+
except Exception:
|
| 62 |
+
return False
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _load_engine():
|
| 66 |
+
"""Build and cache the TTSInferenceEngine. Runs on the GPU worker."""
|
| 67 |
+
global _ENGINE, _SAMPLE_RATE
|
| 68 |
+
if _ENGINE is not None:
|
| 69 |
+
return _ENGINE
|
| 70 |
+
|
| 71 |
+
import torch
|
| 72 |
+
from huggingface_hub import snapshot_download
|
| 73 |
+
from fish_speech.models.text2semantic.inference import launch_thread_safe_queue
|
| 74 |
+
from fish_speech.models.dac.inference import load_model as load_decoder_model
|
| 75 |
+
from fish_speech.inference_engine import TTSInferenceEngine
|
| 76 |
+
|
| 77 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 78 |
+
precision = torch.half if device == "cuda" else torch.float32
|
| 79 |
+
|
| 80 |
+
checkpoint_dir = snapshot_download(repo_id=TTS_MODEL_REPO)
|
| 81 |
+
|
| 82 |
+
llama_queue = launch_thread_safe_queue(
|
| 83 |
+
checkpoint_path=checkpoint_dir,
|
| 84 |
+
device=device,
|
| 85 |
+
precision=precision,
|
| 86 |
+
compile=False,
|
| 87 |
+
)
|
| 88 |
+
decoder_model = load_decoder_model(
|
| 89 |
+
config_name=DECODER_CONFIG,
|
| 90 |
+
checkpoint_path=os.path.join(checkpoint_dir, DECODER_CHECKPOINT),
|
| 91 |
+
device=device,
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
engine = TTSInferenceEngine(
|
| 95 |
+
llama_queue=llama_queue,
|
| 96 |
+
decoder_model=decoder_model,
|
| 97 |
+
compile=False,
|
| 98 |
+
precision=precision,
|
| 99 |
+
)
|
| 100 |
+
try:
|
| 101 |
+
_SAMPLE_RATE = int(decoder_model.sample_rate)
|
| 102 |
+
except Exception:
|
| 103 |
+
_SAMPLE_RATE = 44100
|
| 104 |
+
_ENGINE = engine
|
| 105 |
+
return engine
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _build_request(text: str, voice: VoiceConfig):
|
| 109 |
+
from fish_speech.utils.schema import ServeTTSRequest, ServeReferenceAudio
|
| 110 |
+
|
| 111 |
+
references = []
|
| 112 |
+
if voice.ref_audio and os.path.isfile(voice.ref_audio):
|
| 113 |
+
with open(voice.ref_audio, "rb") as f:
|
| 114 |
+
audio_bytes = f.read()
|
| 115 |
+
references = [ServeReferenceAudio(audio=audio_bytes, text=voice.ref_text or "")]
|
| 116 |
+
|
| 117 |
+
return ServeTTSRequest(
|
| 118 |
+
text=text,
|
| 119 |
+
references=references,
|
| 120 |
+
reference_id=None,
|
| 121 |
+
max_new_tokens=1024,
|
| 122 |
+
chunk_length=200,
|
| 123 |
+
top_p=0.8,
|
| 124 |
+
repetition_penalty=1.1,
|
| 125 |
+
temperature=0.8,
|
| 126 |
+
format="wav",
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
@GPU(duration=120)
|
| 131 |
+
def synthesize(text: str, voice: VoiceConfig) -> Tuple[int, np.ndarray]:
|
| 132 |
+
"""Synthesize one utterance. Returns (sample_rate, float32 mono waveform)."""
|
| 133 |
+
engine = _load_engine()
|
| 134 |
+
request = _build_request(text, voice)
|
| 135 |
+
|
| 136 |
+
audio_chunks: List[np.ndarray] = []
|
| 137 |
+
sample_rate = _SAMPLE_RATE
|
| 138 |
+
for result in engine.inference(request):
|
| 139 |
+
code = getattr(result, "code", None)
|
| 140 |
+
if code == "final" and getattr(result, "audio", None) is not None:
|
| 141 |
+
sample_rate, audio = result.audio
|
| 142 |
+
audio_chunks.append(np.asarray(audio, dtype=np.float32).reshape(-1))
|
| 143 |
+
elif code == "error":
|
| 144 |
+
raise RuntimeError(f"TTS inference error: {getattr(result, 'error', 'unknown')}")
|
| 145 |
+
|
| 146 |
+
if not audio_chunks:
|
| 147 |
+
raise RuntimeError("TTS produced no audio.")
|
| 148 |
+
return int(sample_rate), np.concatenate(audio_chunks)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@GPU(duration=300)
|
| 152 |
+
def generate_podcast(
|
| 153 |
+
lines: List[Tuple[str, str]],
|
| 154 |
+
voice_map: dict,
|
| 155 |
+
*,
|
| 156 |
+
gap_seconds: float = 0.4,
|
| 157 |
+
progress=None,
|
| 158 |
+
) -> Tuple[int, np.ndarray]:
|
| 159 |
+
"""Synthesize each (speaker, text) line and stitch into one waveform.
|
| 160 |
+
|
| 161 |
+
``voice_map`` maps speaker name -> VoiceConfig. The whole loop runs inside a single
|
| 162 |
+
GPU allocation so the model is loaded once per podcast.
|
| 163 |
+
"""
|
| 164 |
+
engine = _load_engine()
|
| 165 |
+
segments: List[np.ndarray] = []
|
| 166 |
+
sample_rate = _SAMPLE_RATE
|
| 167 |
+
default_voice = VoiceConfig()
|
| 168 |
+
|
| 169 |
+
total = len(lines)
|
| 170 |
+
for i, (speaker, text) in enumerate(lines):
|
| 171 |
+
if not text.strip():
|
| 172 |
+
continue
|
| 173 |
+
if progress is not None:
|
| 174 |
+
progress((i / max(total, 1)), desc=f"Voicing line {i + 1}/{total} ({speaker})")
|
| 175 |
+
voice = voice_map.get(speaker, default_voice)
|
| 176 |
+
request = _build_request(text, voice)
|
| 177 |
+
for result in engine.inference(request):
|
| 178 |
+
if getattr(result, "code", None) == "final" and getattr(result, "audio", None):
|
| 179 |
+
sample_rate, audio = result.audio
|
| 180 |
+
segments.append(np.asarray(audio, dtype=np.float32).reshape(-1))
|
| 181 |
+
if gap_seconds > 0:
|
| 182 |
+
segments.append(np.zeros(int(sample_rate * gap_seconds), dtype=np.float32))
|
| 183 |
+
|
| 184 |
+
if not segments:
|
| 185 |
+
raise RuntimeError("No audio was generated for this script.")
|
| 186 |
+
return int(sample_rate), np.concatenate(segments)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def write_wav(sample_rate: int, audio: np.ndarray) -> str:
|
| 190 |
+
"""Write a waveform to a temp .wav file and return its path (for download)."""
|
| 191 |
+
import soundfile as sf
|
| 192 |
+
|
| 193 |
+
path = tempfile.mktemp(suffix=".wav")
|
| 194 |
+
sf.write(path, audio, sample_rate)
|
| 195 |
+
return path
|
tts/voices.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Preset voice registry.
|
| 2 |
+
|
| 3 |
+
OpenAudio S1-mini does zero-shot cloning, so a "preset" is simply a curated short
|
| 4 |
+
reference clip plus its transcript. Drop matching ``<id>.wav`` / ``<id>.txt`` files into
|
| 5 |
+
``tts/voices/`` and they are picked up automatically; entries declared in ``PRESETS``
|
| 6 |
+
without a bundled clip act as "model default" voices (no reference passed).
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import os
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Dict, List, Optional
|
| 14 |
+
|
| 15 |
+
_VOICES_DIR = os.path.join(os.path.dirname(__file__), "voices")
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class Voice:
|
| 20 |
+
id: str
|
| 21 |
+
label: str
|
| 22 |
+
ref_audio: Optional[str] # path to reference .wav, or None for model default
|
| 23 |
+
ref_text: str = ""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# Declared presets. ref_audio is resolved from tts/voices/<id>.wav if present.
|
| 27 |
+
PRESETS: List[Dict[str, str]] = [
|
| 28 |
+
{"id": "default", "label": "Model default (no clone)"},
|
| 29 |
+
{"id": "narrator_warm", "label": "Warm Narrator"},
|
| 30 |
+
{"id": "host_energetic", "label": "Energetic Host"},
|
| 31 |
+
{"id": "guest_calm", "label": "Calm Guest"},
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _resolve(voice_id: str) -> Voice:
|
| 36 |
+
label = next((p["label"] for p in PRESETS if p["id"] == voice_id), voice_id)
|
| 37 |
+
wav = os.path.join(_VOICES_DIR, f"{voice_id}.wav")
|
| 38 |
+
txt = os.path.join(_VOICES_DIR, f"{voice_id}.txt")
|
| 39 |
+
ref_audio = wav if os.path.isfile(wav) else None
|
| 40 |
+
ref_text = ""
|
| 41 |
+
if os.path.isfile(txt):
|
| 42 |
+
with open(txt, "r", encoding="utf-8") as f:
|
| 43 |
+
ref_text = f.read().strip()
|
| 44 |
+
return Voice(id=voice_id, label=label, ref_audio=ref_audio, ref_text=ref_text)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def list_presets() -> List[str]:
|
| 48 |
+
"""Labels for the UI dropdown."""
|
| 49 |
+
return [p["label"] for p in PRESETS]
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def label_to_id(label: str) -> str:
|
| 53 |
+
for p in PRESETS:
|
| 54 |
+
if p["label"] == label:
|
| 55 |
+
return p["id"]
|
| 56 |
+
return "default"
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def get_voice(label_or_id: str) -> Voice:
|
| 60 |
+
vid = label_to_id(label_or_id)
|
| 61 |
+
if vid == label_or_id and vid not in {p["id"] for p in PRESETS}:
|
| 62 |
+
vid = "default"
|
| 63 |
+
return _resolve(vid)
|
tts/voices/README.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Preset reference clips
|
| 2 |
+
|
| 3 |
+
Drop voice presets here as pairs:
|
| 4 |
+
|
| 5 |
+
- `<id>.wav` — a clean 10–30s reference clip of the target voice
|
| 6 |
+
- `<id>.txt` — the exact transcript of that clip (improves cloning quality)
|
| 7 |
+
|
| 8 |
+
The `<id>` must match an entry in `tts/voices.py::PRESETS` (e.g. `narrator_warm.wav`).
|
| 9 |
+
A preset declared in `PRESETS` without a `.wav` here falls back to the model's default
|
| 10 |
+
voice (no cloning reference). `.wav` files are tracked via Git LFS (see `.gitattributes`).
|