Claw commited on
Commit ·
5c0fc49
1
Parent(s): 0e16ab7
Fix make_dub filter_complex: don't include silent base as filter, only delay TTS segments
Browse files
app.py
CHANGED
|
@@ -47,7 +47,6 @@ def load_asr():
|
|
| 47 |
|
| 48 |
def transcribe(wav):
|
| 49 |
processor, model = load_asr()
|
| 50 |
-
# Use soundfile to load (avoids torchcodec issues)
|
| 51 |
audio, sr = sf.read(str(wav))
|
| 52 |
if len(audio.shape) > 1:
|
| 53 |
audio = audio.mean(axis=1)
|
|
@@ -112,11 +111,9 @@ def load_tts():
|
|
| 112 |
|
| 113 |
def tts(text, wav):
|
| 114 |
tokenizer, model = load_tts()
|
| 115 |
-
# Add attention_mask to avoid warning
|
| 116 |
inputs = tokenizer(text, return_tensors="pt", padding=True)
|
| 117 |
with torch.inference_mode():
|
| 118 |
audio = model(**inputs).waveform.cpu()
|
| 119 |
-
# Use soundfile instead of torchaudio.save (avoids torchcodec)
|
| 120 |
sf.write(str(wav), audio.squeeze().numpy(), model.config.sampling_rate)
|
| 121 |
|
| 122 |
def fit_audio(src, dst, target):
|
|
@@ -140,12 +137,14 @@ def make_dub(ms, total, work):
|
|
| 140 |
weights = [max(1,len(re.sub(r'\s+','',x))) for x in parts]
|
| 141 |
total_w = sum(weights)
|
| 142 |
|
|
|
|
| 143 |
silent = work/"silent.wav"
|
| 144 |
run(["ffmpeg","-y","-f","lavfi","-i","anullsrc=r=24000:cl=mono",
|
| 145 |
"-t",f"{total:.3f}","-ar","24000","-ac","1",str(silent)])
|
| 146 |
|
| 147 |
-
|
| 148 |
-
|
|
|
|
| 149 |
start = 0.0
|
| 150 |
|
| 151 |
for i, part in enumerate(parts, 1):
|
|
@@ -153,13 +152,16 @@ def make_dub(ms, total, work):
|
|
| 153 |
raw, fitted = work/f"tts_{i}.wav", work/f"fit_{i}.wav"
|
| 154 |
tts(part, raw)
|
| 155 |
fit_audio(raw, fitted, slot)
|
| 156 |
-
inputs += ["-i",str(fitted)]
|
| 157 |
delay = int(start*1000)
|
| 158 |
-
|
| 159 |
start += slot
|
| 160 |
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
| 163 |
out = work/"dub.wav"
|
| 164 |
run(["ffmpeg","-y",*inputs,"-filter_complex",filt,"-map","[a]",
|
| 165 |
"-ar","24000","-ac","1",str(out)])
|
|
|
|
| 47 |
|
| 48 |
def transcribe(wav):
|
| 49 |
processor, model = load_asr()
|
|
|
|
| 50 |
audio, sr = sf.read(str(wav))
|
| 51 |
if len(audio.shape) > 1:
|
| 52 |
audio = audio.mean(axis=1)
|
|
|
|
| 111 |
|
| 112 |
def tts(text, wav):
|
| 113 |
tokenizer, model = load_tts()
|
|
|
|
| 114 |
inputs = tokenizer(text, return_tensors="pt", padding=True)
|
| 115 |
with torch.inference_mode():
|
| 116 |
audio = model(**inputs).waveform.cpu()
|
|
|
|
| 117 |
sf.write(str(wav), audio.squeeze().numpy(), model.config.sampling_rate)
|
| 118 |
|
| 119 |
def fit_audio(src, dst, target):
|
|
|
|
| 137 |
weights = [max(1,len(re.sub(r'\s+','',x))) for x in parts]
|
| 138 |
total_w = sum(weights)
|
| 139 |
|
| 140 |
+
# Create silent base
|
| 141 |
silent = work/"silent.wav"
|
| 142 |
run(["ffmpeg","-y","-f","lavfi","-i","anullsrc=r=24000:cl=mono",
|
| 143 |
"-t",f"{total:.3f}","-ar","24000","-ac","1",str(silent)])
|
| 144 |
|
| 145 |
+
# Generate and fit each TTS segment
|
| 146 |
+
inputs = ["-i", str(silent)]
|
| 147 |
+
filter_parts = []
|
| 148 |
start = 0.0
|
| 149 |
|
| 150 |
for i, part in enumerate(parts, 1):
|
|
|
|
| 152 |
raw, fitted = work/f"tts_{i}.wav", work/f"fit_{i}.wav"
|
| 153 |
tts(part, raw)
|
| 154 |
fit_audio(raw, fitted, slot)
|
| 155 |
+
inputs += ["-i", str(fitted)]
|
| 156 |
delay = int(start*1000)
|
| 157 |
+
filter_parts.append(f"[{i}:a]adelay={delay}|{delay}[d{i}]")
|
| 158 |
start += slot
|
| 159 |
|
| 160 |
+
# Build filter_complex: delay each TTS segment, then mix with silent base
|
| 161 |
+
# Input 0 = silent, Input 1..n = TTS segments
|
| 162 |
+
mix_inputs = "[0:a]" + "".join(f"[d{i}]" for i in range(1, len(parts)+1))
|
| 163 |
+
filt = ";".join(filter_parts) + f";{mix_inputs}amix=inputs={len(parts)+1}:duration=first[a]"
|
| 164 |
+
|
| 165 |
out = work/"dub.wav"
|
| 166 |
run(["ffmpeg","-y",*inputs,"-filter_complex",filt,"-map","[a]",
|
| 167 |
"-ar","24000","-ac","1",str(out)])
|