Delete rag_pipeline.py

rag_pipeline.py

@@ -1,57 +0,0 @@
-import pandas as pd
-import faiss
-import numpy as np
-from langdetect import detect
-from sentence_transformers import SentenceTransformer
-from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
-import zipfile
-import os
-
-# Extract zip if needed
-zip_file = "all_languages_test.csv.zip"
-csv_file = "all_languages_test.csv"
-
-if not os.path.exists(csv_file):
-    with zipfile.ZipFile(zip_file, "r") as zip_ref:
-        zip_ref.extractall()
-
-# Translation model
-tokenizer = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
-model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
-
-lang_map = {
-    "en": "en_XX", "fr": "fr_XX", "ur": "ur_PK", "hi": "hi_IN",
-    "es": "es_XX", "de": "de_DE", "zh-cn": "zh_CN", "ar": "ar_AR"
-}
-
-@st.cache_resource
-def load_index():
-    df = pd.read_csv("all_languages_test.csv").dropna()
-    df["context"] = df["premise"] + " " + df["hypothesis"]
-    corpus = df["context"].tolist()
-
-    embedder = SentenceTransformer("distiluse-base-multilingual-cased-v2")
-    embeddings = embedder.encode(corpus, show_progress_bar=True)
-    
-    index = faiss.IndexFlatL2(embeddings.shape[1])
-    index.add(np.array(embeddings))
-    
-    return embedder, index, corpus
-
-def generate_answer(query, k=3):
-    lang = detect(query)
-    token_lang = lang_map.get(lang, "en_XX")
-    
-    embedder, index, corpus = load_index()
-    
-    query_vec = embedder.encode([query])
-    D, I = index.search(np.array(query_vec), k)
-    contexts = [corpus[i] for i in I[0]]
-    context = " ".join(contexts)
-
-    full_input = f"question: {query} context: {context}"
-    tokenizer.src_lang = token_lang
-    encoded_input = tokenizer(full_input, return_tensors="pt")
-    generated_tokens = model.generate(**encoded_input, forced_bos_token_id=tokenizer.lang_code_to_id[token_lang])
-    
-    return tokenizer.decode(generated_tokens[0], skip_special_tokens=True)