Update app.py

app.py

@@ -9,7 +9,28 @@ import os
 import tempfile
 import chromadb
 
-# Chroma DB initialization
+# ✅ Chunked Upload Config (50 MB chunks to bypass 200 MB limit)
+MAX_CHUNK_SIZE = 50 * 1024 * 1024  # 50 MB
+
+def upload_file_in_chunks(file):
+    """Split file into 50 MB chunks to bypass the 200 MB limit."""
+    chunk_list = []
+    bytes_data = file.read()
+    total_size = len(bytes_data)
+    num_chunks = (total_size // MAX_CHUNK_SIZE) + 1
+
+    st.write(f"🔹 Splitting into {num_chunks} chunks...")
+
+    for i in range(num_chunks):
+        start = i * MAX_CHUNK_SIZE
+        end = min((i + 1) * MAX_CHUNK_SIZE, total_size)
+        chunk_list.append(bytes_data[start:end])
+
+    # Combine all chunks into a single CSV
+    full_csv_data = b"".join(chunk_list)
+    return full_csv_data
+
+# Initialize Chroma DB
 chroma_client = chromadb.Client()
 collection_name = "subtitle_chunks"
 
@@ -22,7 +43,7 @@ def extract_audio(video_file, audio_file):
     output_file = ffmpeg.output(input_file, audio_file, **{'vn': None, 'ar': 16000, 'ac': 1, 'f': 'wav'})
     ffmpeg.run(output_file)
 
-# Function to transcribe audio using Hugging Face Whisper
+# Function to transcribe audio using Whisper
 def transcribe_audio(audio_file):
     result = whisper_model(audio_file)
     return result['text']
@@ -41,7 +62,7 @@ def split_csv(csv_path, chunk_size=50000):
         chunks.append(chunk_file)
     return chunks
 
-# Function to store subtitle chunks in Chroma DB
+# Store subtitle chunks in Chroma DB
 def store_chunks_in_chroma(chunk_files):
     collection = chroma_client.create_collection(name=collection_name)
 
@@ -58,7 +79,7 @@ def store_chunks_in_chroma(chunk_files):
                 ids=[f"{chunk_file}_{idx}"]
             )
 
-# Function to search subtitles using Chroma DB
+# Search subtitles in Chroma DB
 def search_in_chroma(transcribed_text, top_k=10):
     query_embedding = generate_embedding(transcribed_text)
     collection = chroma_client.get_collection(name=collection_name)
@@ -77,12 +98,27 @@ def search_in_chroma(transcribed_text, top_k=10):
 # Streamlit UI
 st.title("🎥 Video Subtitle Search with Hugging Face Whisper + Chroma DB")
 
+# Sidebar for CSV Upload with Chunking
+st.sidebar.header("📁 Upload Subtitle Database")
+csv_file = st.sidebar.file_uploader("Upload Subtitle CSV (200 MB+ supported)", type=["csv"])
+
 # Upload video
-uploaded_file = st.file_uploader("Upload a video", type=["mp4", "avi", "mov", "mkv"])
+uploaded_video = st.file_uploader("Upload a video", type=["mp4", "avi", "mov", "mkv"])
+
+# If both CSV and video are uploaded
+if csv_file and uploaded_video:
+    # Handle large CSV upload using 50 MB chunks
+    st.info("Uploading CSV in 50 MB chunks...")
+    full_csv_data = upload_file_in_chunks(csv_file)
 
-if uploaded_file:
+    # Save CSV locally
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as temp_csv:
+        temp_csv.write(full_csv_data)
+        csv_path = temp_csv.name
+
+    # Save video locally
     with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_video:
-        temp_video.write(uploaded_file.getbuffer())
+        temp_video.write(uploaded_video.getbuffer())
         video_path = temp_video.name
 
     audio_path = "temp_audio.wav"
@@ -96,10 +132,9 @@ if uploaded_file:
     transcribed_text = transcribe_audio(audio_path)
     st.text_area("Transcribed Text", transcribed_text, height=150)
 
-    # Split the large CSV database into smaller chunks
-    st.info("Splitting large database into smaller chunks...")
-    subtitle_db_path = "database.csv"
-    chunk_files = split_csv(subtitle_db_path, chunk_size=50000)
+    # Split the CSV database into smaller chunks
+    st.info("Splitting CSV into smaller chunks...")
+    chunk_files = split_csv(csv_path, chunk_size=50000)
 
     # Store chunks in Chroma DB
     st.info("Storing subtitle chunks in Chroma DB...")
@@ -122,7 +157,11 @@ if uploaded_file:
     # Cleanup
     os.remove(video_path)
     os.remove(audio_path)
+    os.remove(csv_path)
 
     # Remove chunk files
     for chunk_file in chunk_files:
         os.remove(chunk_file)
+
+else:
+    st.warning("Please upload both a video file and a CSV file to proceed.")