Update app.py

app.py

@@ -15,16 +15,24 @@ from collections import deque
 os.environ["COQUI_TOS_AGREED"] = "1"
 
 # =========================
-# 🔥 LOAD MODEL ONCE
+# 🔥 DEVICE
 # =========================
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
-print("🚀 Loading XTTS model...")
-tts = TTS(
-    model_name="tts_models/multilingual/multi-dataset/xtts_v2",
-    progress_bar=False
-).to(device)
-print("✅ Model loaded!")
+# =========================
+# 🔥 MULTI-CHANNEL MODEL POOL
+# =========================
+NUM_CHANNELS = 5
+print(f"🚀 Loading {NUM_CHANNELS} XTTS channels...")
+
+channels = []
+for i in range(NUM_CHANNELS):
+    tts_model = TTS(
+        model_name="tts_models/multilingual/multi-dataset/xtts_v2",
+        progress_bar=False
+    ).to(device)
+    channels.append({"tts": tts_model, "busy": False})
+    print(f"✅ Channel {i+1} loaded")
 
 # =========================
 # 📁 OUTPUT DIR
@@ -33,48 +41,41 @@ OUTPUT_DIR = "outputs"
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 # =========================
-# ⚡ BATCH CONFIG
+# 🔹 REQUEST QUEUE
 # =========================
-BATCH_SIZE = 5
-BATCH_WAIT_TIME = 2  # seconds
-
 request_queue = deque()
 
 # =========================
-# 🔥 BATCH WORKER
+# 🔥 MULTI-CHANNEL WORKER
 # =========================
-async def batch_worker():
-    print("🔥 Batch worker started...")
-
+async def channel_worker(channel):
+    """Worker that processes requests assigned to this channel."""
     while True:
         if len(request_queue) == 0:
             await asyncio.sleep(0.1)
             continue
 
-        # Wait to collect batch
-        await asyncio.sleep(BATCH_WAIT_TIME)
-
-        batch = []
-        while len(request_queue) > 0 and len(batch) < BATCH_SIZE:
-            batch.append(request_queue.popleft())
-
-        print(f"⚡ Processing batch of {len(batch)}")
-
-        for item in batch:
-            text, lang, audio_path, output_path, future = item
+        if channel["busy"]:
+            await asyncio.sleep(0.05)
+            continue
 
-            try:
-                tts.tts_to_file(
-                    text=text,
-                    speaker_wav=audio_path,
-                    language=lang,
-                    file_path=output_path,
-                    split_sentences=True
-                )
-                future.set_result(output_path)
+        # Pick next request
+        text, lang, audio_path, output_path, future = request_queue.popleft()
+        channel["busy"] = True
 
-            except Exception as e:
-                future.set_result(str(e))
+        try:
+            channel["tts"].tts_to_file(
+                text=text,
+                speaker_wav=audio_path,
+                language=lang,
+                file_path=output_path,
+                split_sentences=True
+            )
+            future.set_result(output_path)
+        except Exception as e:
+            future.set_result(str(e))
+        finally:
+            channel["busy"] = False
 
 
 # =========================
@@ -84,7 +85,10 @@ api = FastAPI()
 
 @api.on_event("startup")
 async def startup_event():
-    asyncio.create_task(batch_worker())
+    # Start a worker for each channel
+    for ch in channels:
+        asyncio.create_task(channel_worker(ch))
+    print(f"🔥 {NUM_CHANNELS} channel workers started!")
 
 
 @api.post("/clone-voice/")
@@ -103,6 +107,7 @@ async def clone_voice_api(
         loop = asyncio.get_event_loop()
         future = loop.create_future()
 
+        # Add to request queue
         request_queue.append((text, language, input_path, output_path, future))
 
         result = await future
@@ -132,7 +137,6 @@ async def clone_voice_ui(audio_path, text, language):
     future = loop.create_future()
 
     request_queue.append((text, language, audio_path, output_path, future))
-
     result = await future
 
     if isinstance(result, str) and result.endswith(".wav"):
@@ -141,8 +145,8 @@ async def clone_voice_ui(audio_path, text, language):
         return f"❌ {result}", None
 
 
-with gr.Blocks(title="XTTS Voice Cloning (Batching)") as demo:
-    gr.Markdown("# 🎤 XTTS Voice Cloning (Batch Mode)")
+with gr.Blocks(title="XTTS Voice Cloning (Multi-Channel)") as demo:
+    gr.Markdown("# 🎤 XTTS Voice Cloning (Multi-Channel)")
 
     audio_input = gr.Audio(type="filepath", label="Speaker Audio")
     text_input = gr.Textbox(label="Text")
@@ -159,7 +163,6 @@ with gr.Blocks(title="XTTS Voice Cloning (Batching)") as demo:
         outputs=[status, output_audio]
     )
 
-# ✅ FIXED QUEUE (no concurrency_count)
 demo.queue(max_size=20)
 
 # =========================