Update app.py

app.py

@@ -1,290 +1,263 @@
-# app.py - Hybrid Memory-Augmented Graph RAG with Groq
-import hashlib
+# app.py - Hybrid RAG (FAISS semantic + BM25, fused with RRF) for multi-format documents.
+# Handles: PDFs (text + tables), DOCX (with tables), PPTX, XLSX/CSV, TXT/MD. Arabic + English.
+import io
 import os
-import pickle
+import re
+import unicodedata
 from collections import defaultdict
-from datetime import datetime
 
-import networkx as nx
+import faiss
+import fitz  # PyMuPDF
 import numpy as np
-import spacy
+import pandas as pd
 import streamlit as st
-from langchain_community.vectorstores import FAISS
-from langchain_huggingface import HuggingFaceEmbeddings
+from docx import Document
 from langchain_groq import ChatGroq
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from pypdf import PdfReader
+from pptx import Presentation
 from rank_bm25 import BM25Okapi
-from sentence_transformers import CrossEncoder, SentenceTransformer
-from sklearn.metrics.pairwise import cosine_similarity
-
-# For DOCX support
-try:
-    from docx import Document
-except ImportError:
-    pass  # Will show warning if missing
+from sentence_transformers import SentenceTransformer
 
 # ========================= CONSTANTS =========================
-EMBED_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
-RERANKER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+EMBED_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"  # Arabic + English, 384-dim
 LLM_MODEL_NAME = "llama-3.3-70b-versatile"
-DEFAULT_TOP_K = 5
+CHUNK_WORDS = 80          # stays under the embedding model's 128-token limit
+CHUNK_OVERLAP = 15
+TABLE_CHUNK_WORDS = 60    # table/spreadsheet rows are token-heavy, so smaller groups
 RRF_K = 60
 
-# ========================= HELPERS =========================
-def get_groq_api_key():
-    try:
-        return st.secrets.get("GROQ_API_KEY") or os.getenv("GROQ_API_KEY")
-    except:
-        return os.getenv("GROQ_API_KEY")
-
-def rrf_score(rank: int, k: int = RRF_K) -> float:
-    return 1.0 / (k + rank)
-
-# ========================= MEMORY LAYER =========================
-# ========================= MEMORY LAYER =========================
-class HybridMemory:
-    def __init__(self):
-        self.embeddings = HuggingFaceEmbeddings(
-            model_name=EMBED_MODEL_NAME,
-            cache_folder="/tmp/model_cache",
-            model_kwargs={"device": "cpu"},
-        )
-        self.vector_memory = None
-        self.graph_memory = nx.Graph()
-        self.memory_dir = "user_memory"
-        os.makedirs(self.memory_dir, exist_ok=True)
-        self.load_memory()
-
-    def load_memory(self):
-        vector_path = os.path.join(self.memory_dir, "vector_memory")
-        graph_path = os.path.join(self.memory_dir, "memory_graph.pkl")
-        if os.path.exists(vector_path):
-            self.vector_memory = FAISS.load_local(vector_path, self.embeddings, allow_dangerous_deserialization=True)
-        if os.path.exists(graph_path):
-            with open(graph_path, "rb") as f:
-                self.graph_memory = pickle.load(f)
-
-    def save_memory(self):
-        if self.vector_memory:
-            self.vector_memory.save_local(os.path.join(self.memory_dir, "vector_memory"))
-        with open(os.path.join(self.memory_dir, "memory_graph.pkl"), "wb") as f:
-            pickle.dump(self.graph_memory, f)
-
-    def add_memory(self, text: str, memory_type: str = "fact", metadata: dict = None):
-        if metadata is None:
-            metadata = {}
-        metadata.update({
-            "timestamp": datetime.now().isoformat(),
-            "type": memory_type,
-        })
-
-        # Add to Vector Memory
-        if self.vector_memory is None:
-            self.vector_memory = FAISS.from_texts([text], self.embeddings, metadatas=[metadata])
-        else:
-            self.vector_memory.add_texts([text], metadatas=[metadata])
-
-        # Add to Graph Memory
-        self._add_to_graph(text)
-        self.save_memory()
-
-    def _add_to_graph(self, text: str):
-        """Safely add entities to graph"""
-        try:
-            # Get nlp from session_state or load it directly
-            nlp = st.session_state.get("nlp")
-            if not nlp:
-                nlp = load_spacy()  # fallback
-                st.session_state.nlp = nlp
-
-            doc = nlp(text[:30000])
-            entities = [ent.text.strip().lower() for ent in doc.ents if len(ent.text.strip()) > 1]
-            
-            if not entities:
-                return  # No entities found
-
-            for ent in entities:
-                if not self.graph_memory.has_node(ent):
-                    self.graph_memory.add_node(ent, count=1)
-                else:
-                    self.graph_memory.nodes[ent]["count"] = self.graph_memory.nodes[ent].get("count", 0) + 1
-
-            for i in range(len(entities)):
-                for j in range(i + 1, len(entities)):
-                    a, b = entities[i], entities[j]
-                    if self.graph_memory.has_edge(a, b):
-                        self.graph_memory[a][b]["weight"] += 1
-                    else:
-                        self.graph_memory.add_edge(a, b, weight=1)
-        except Exception as e:
-            # st.warning(f"Graph memory update skipped: {e}")  # uncomment for debug
-            pass
-
-    def retrieve_memory(self, query: str, top_k: int = 6):
-        if not self.vector_memory:
-            return []
-        docs = self.vector_memory.similarity_search(query, k=top_k)
-        return [{"text": doc.page_content, "metadata": doc.metadata} for doc in docs]
-
-    def get_stats(self):
-        return {
-            "vector_memories": len(self.vector_memory.index_to_docstore_id) if self.vector_memory else 0,
-            "graph_nodes": self.graph_memory.number_of_nodes(),
-            "graph_edges": self.graph_memory.number_of_edges()
-        }
-
-# ========================= ORIGINAL PIPELINE FUNCTIONS =========================
-@st.cache_resource
-def load_embed_model():
-    return SentenceTransformer(EMBED_MODEL_NAME, cache_folder="/tmp/model_cache")
-
-@st.cache_resource
-def load_reranker():
-    return CrossEncoder(RERANKER_MODEL)
-
-@st.cache_resource
-def load_lc_embeddings():
-    return HuggingFaceEmbeddings(
-        model_name=EMBED_MODEL_NAME,
-        cache_folder="/tmp/model_cache",
-        model_kwargs={"device": "cpu"},
-    )
-
-@st.cache_resource
-def load_spacy():
+# ========================= TEXT NORMALIZATION =========================
+DIACRITICS_RE = re.compile(r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06ED]")
+DIGIT_MAP = str.maketrans("٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹", "01234567890123456789")
+
+def light_clean(text):
+    text = unicodedata.normalize("NFKC", text)
+    text = DIACRITICS_RE.sub("", text)
+    return text.replace("\u0640", "")
+
+def normalize(text):
+    text = light_clean(text).translate(DIGIT_MAP)
+    text = re.sub("[إأآٱ]", "ا", text)
+    text = text.replace("ى", "ي").replace("ة", "ه")
+    return text.lower()
+
+def tokenize(text):
+    tokens = re.findall(r"[\w\+#]+", normalize(text))
+    return [t[2:] if len(t) > 4 and t.startswith("ال") else t for t in tokens]
+
+# ========================= EXTRACTION =========================
+def make_block(text, source, loc, kind):
+    return {"text": text, "source": source, "loc": loc, "kind": kind}
+
+def pdf_tables(page):
+    out = []
     try:
-        return spacy.load("en_core_web_sm")
-    except OSError:
-        from spacy.cli import download
-        download("en_core_web_sm")
-        return spacy.load("en_core_web_sm")
-
-def load_documents(uploaded_files):
-    """Support PDF, TXT, and DOCX"""
-    docs = []
-    for f in uploaded_files:
-        text = ""
-        source = f.name
-        
-        try:
-            if f.name.lower().endswith('.pdf'):
-                reader = PdfReader(f)
-                text = "\n".join([page.extract_text() or "" for page in reader.pages])
-            elif f.name.lower().endswith('.txt'):
-                text = f.read().decode("utf-8")
-            elif f.name.lower().endswith(('.docx', '.doc')):
-                doc = Document(f)
-                text = "\n".join([paragraph.text for paragraph in doc.paragraphs])
-            else:
-                st.warning(f"Unsupported file type: {f.name}")
+        for t in page.find_tables().tables:
+            lines = []
+            for r in t.extract():
+                cells = [(c or "").replace("\n", " ").strip() for c in r]
+                if any(cells):
+                    lines.append(" | ".join(cells))
+            if len(lines) >= 2:
+                out.append("\n".join(lines))
+    except Exception:
+        pass
+    return out
+
+def extract_pdf(name, data):
+    blocks = []
+    stats = {"pages": 0, "tables": 0, "notes": []}
+    doc = fitz.open(stream=data, filetype="pdf")
+    empty_pages = []
+    for i, page in enumerate(doc, 1):
+        stats["pages"] += 1
+        loc = f"p.{i}"
+        text = unicodedata.normalize("NFKC", page.get_text("text") or "").strip()
+        for tbl in pdf_tables(page):
+            blocks.append(make_block(tbl, name, loc, "table"))
+            stats["tables"] += 1
+        if len(text) < 50:
+            empty_pages.append(i)
+        if text:
+            blocks.append(make_block(text, name, loc, "text"))
+    if empty_pages:
+        shown = ", ".join(map(str, empty_pages[:15])) + ("..." if len(empty_pages) > 15 else "")
+        stats["notes"].append(f"Little or no text on page(s) {shown}. Scanned pages cannot be read by this app.")
+    return blocks, stats
+
+def extract_docx(name, data):
+    doc = Document(io.BytesIO(data))
+    blocks, tables = [], 0
+    text = "\n".join(p.text for p in doc.paragraphs if p.text.strip())
+    if text.strip():
+        blocks.append(make_block(text, name, None, "text"))
+    for table in doc.tables:
+        lines = []
+        for row in table.rows:
+            cells = [c.text.replace("\n", " ").strip() for c in row.cells]
+            if any(cells):
+                lines.append(" | ".join(cells))
+        if len(lines) >= 2:
+            blocks.append(make_block("\n".join(lines), name, None, "table"))
+            tables += 1
+    return blocks, {"pages": 1, "tables": tables,
+                    "notes": ["Images embedded inside the DOCX are not read."]}
+
+def extract_pptx(name, data):
+    prs = Presentation(io.BytesIO(data))
+    blocks, tables = [], 0
+    for i, slide in enumerate(prs.slides, 1):
+        loc = f"slide {i}"
+        texts = []
+        for shape in slide.shapes:
+            if shape.has_text_frame and shape.text_frame.text.strip():
+                texts.append(shape.text_frame.text.strip())
+            if getattr(shape, "has_table", False) and shape.has_table:
+                lines = [" | ".join(c.text.strip() for c in row.cells) for row in shape.table.rows]
+                if len(lines) >= 2:
+                    blocks.append(make_block("\n".join(lines), name, loc, "table"))
+                    tables += 1
+        if slide.has_notes_slide and slide.notes_slide.notes_text_frame.text.strip():
+            texts.append("Notes: " + slide.notes_slide.notes_text_frame.text.strip())
+        if texts:
+            blocks.append(make_block("\n".join(texts), name, loc, "text"))
+    return blocks, {"pages": len(prs.slides), "tables": tables,
+                    "notes": ["Pictures and charts on slides are not read."]}
+
+def df_to_rows(df):
+    df = df.dropna(how="all").dropna(axis=1, how="all").fillna("")
+    headers = [str(c) for c in df.columns]
+    lines = []
+    for _, row in df.iterrows():
+        cells = []
+        for h, v in zip(headers, row):
+            v = str(v).strip()
+            if v:
+                cells.append(v if h.startswith("Unnamed") else f"{h}: {v}")
+        if cells:
+            lines.append(" | ".join(cells))
+    return "\n".join(lines)
+
+def extract_spreadsheet(name, data):
+    blocks = []
+    if name.lower().endswith(".csv"):
+        df = None
+        for enc in ("utf-8-sig", "cp1256", "latin-1"):
+            try:
+                df = pd.read_csv(io.BytesIO(data), encoding=enc)
+                break
+            except Exception:
                 continue
-        except Exception as e:
-            st.warning(f"Error reading {f.name}: {str(e)}")
-            continue
-
-        if text.strip():
-            docs.append({"text": text, "source": source})
-    return docs
-
-def hybrid_chunking(text, embed_model, base=300, final=1000, thresh=0.6):
-    splitter = RecursiveCharacterTextSplitter(chunk_size=base, chunk_overlap=50)
-    base_chunks = splitter.split_text(text)
-    if not base_chunks: return []
-    embeddings = embed_model.encode(base_chunks, show_progress_bar=False)
-    chunks, current = [], base_chunks[0]
-    anchor = embeddings[0].reshape(1, -1)
-    for i in range(1, len(base_chunks)):
-        next_emb = embeddings[i].reshape(1, -1)
-        sim = cosine_similarity(anchor, next_emb)[0][0]
-        if sim > thresh and len(current) + len(base_chunks[i]) < final:
-            current += " " + base_chunks[i]
+        sheets = {"": df} if df is not None else {}
+    else:
+        sheets = pd.read_excel(io.BytesIO(data), sheet_name=None)
+    for sheet, df in sheets.items():
+        text = df_to_rows(df)
+        if text:
+            blocks.append(make_block(text, name, f"sheet {sheet}" if sheet else None, "rows"))
+    return blocks, {"pages": len(sheets), "tables": len(blocks), "notes": []}
+
+def extract_text_file(name, data):
+    text = data.decode("utf-8", errors="ignore")
+    blocks = [make_block(text, name, None, "text")] if text.strip() else []
+    return blocks, {"pages": 1, "tables": 0, "notes": []}
+
+@st.cache_data(show_spinner=False)
+def extract_file(name, data):
+    lower = name.lower()
+    if lower.endswith(".pdf"):
+        return extract_pdf(name, data)
+    if lower.endswith(".docx"):
+        return extract_docx(name, data)
+    if lower.endswith(".pptx"):
+        return extract_pptx(name, data)
+    if lower.endswith((".xlsx", ".csv")):
+        return extract_spreadsheet(name, data)
+    if lower.endswith((".txt", ".md")):
+        return extract_text_file(name, data)
+    return [], {"pages": 0, "tables": 0, "notes": ["Unsupported file type."]}
+
+# ========================= CHUNKING =========================
+def make_chunk(block, text):
+    return {"text": text, "source": block["source"], "loc": block["loc"], "kind": block["kind"]}
+
+def chunk_blocks(blocks):
+    chunks = []
+    step = max(1, CHUNK_WORDS - CHUNK_OVERLAP)
+    for b in blocks:
+        if b["kind"] in ("table", "rows"):
+            lines = [l for l in b["text"].split("\n") if l.strip()]
+            header = lines[0] if b["kind"] == "table" else None
+            body = lines[1:] if b["kind"] == "table" else lines
+            group, wc = [], 0
+            for line in body:
+                lw = len(line.split())
+                if group and wc + lw > TABLE_CHUNK_WORDS:
+                    chunks.append(make_chunk(b, "\n".join(([header] if header else []) + group)))
+                    group, wc = [], 0
+                group.append(line)
+                wc += lw
+            if group:
+                chunks.append(make_chunk(b, "\n".join(([header] if header else []) + group)))
         else:
-            chunks.append(current)
-            current = base_chunks[i]
-            anchor = next_emb
-    chunks.append(current)
+            words = b["text"].split()
+            for i in range(0, len(words), step):
+                piece = " ".join(words[i:i + CHUNK_WORDS])
+                if len(piece.strip()) > 20:
+                    chunks.append(make_chunk(b, piece))
     return chunks
 
-def chunk_documents(docs, embed_model, method, **kwargs):
-    result = []
-    for doc in docs:
-        if method == "Hybrid (Recursive + Semantic)":
-            texts = hybrid_chunking(doc["text"], embed_model, **kwargs)
-        else:
-            splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-            texts = splitter.split_text(doc["text"])
-        for t in texts:
-            result.append({"text": t, "source": doc["source"]})
-    return result
-
-def build_knowledge_graph(chunks, nlp):
-    G = nx.Graph()
-    entity_to_chunks = defaultdict(list)
-    for idx, chunk in enumerate(chunks):
-        doc = nlp(chunk["text"][:50000])
-        entities = [ent.text.strip().lower() for ent in doc.ents if len(ent.text.strip()) > 1]
-        for e in set(entities):
-            if not G.has_node(e):
-                G.add_node(e)
-            entity_to_chunks[e].append(idx)
-        for i in range(len(entities)):
-            for j in range(i+1, len(entities)):
-                a, b = entities[i], entities[j]
-                if G.has_edge(a, b):
-                    G[a][b]["weight"] += 1
-                else:
-                    G.add_edge(a, b, weight=1)
-    return G, dict(entity_to_chunks)
-
-def build_indexes(chunks, lc_embeddings):
-    texts = [c["text"] for c in chunks]
-    bm25 = BM25Okapi([t.lower().split() for t in texts])
-    faiss_index = FAISS.from_texts(texts, lc_embeddings)
-    chunk_lookup = {c["text"]: c for c in chunks}
-    return bm25, faiss_index, chunk_lookup
-
-def hybrid_memory_retrieve(query, chunks, bm25, faiss_index, chunk_lookup, graph, entity_to_chunks, reranker, nlp, memory, top_k=5, hops=2):
-    texts = [c["text"] for c in chunks]
-    bm25_scores = bm25.get_scores(query.lower().split())
-    bm25_idx = np.argsort(bm25_scores)[::-1][:top_k*3]
-    faiss_docs = faiss_index.similarity_search(query, k=top_k*3)
-
-    candidates = []
-    seen = set()
-    for i in bm25_idx[:top_k*2]:
-        c = chunks[i]
-        if c["text"] not in seen:
-            candidates.append(c)
-            seen.add(c["text"])
-    for doc in faiss_docs:
-        c = chunk_lookup.get(doc.page_content)
-        if c and c["text"] not in seen:
-            candidates.append(c)
-            seen.add(c["text"])
-
-    mem_items = memory.retrieve_memory(query, top_k=top_k*2)
-
-    all_texts = [c["text"] for c in candidates] + [m["text"] for m in mem_items]
-    pairs = [(query, t) for t in all_texts]
-    scores = reranker.predict(pairs)
-
-    scored = sorted(zip(scores, all_texts), reverse=True)
-    final_texts = [t for _, t in scored[:top_k]]
-
-    final_chunks = []
-    for t in final_texts:
-        for c in candidates:
-            if c["text"] == t:
-                final_chunks.append(c)
-                break
-        else:
-            final_chunks.append({"text": t, "source": "🧠 Memory"})
-    return final_chunks[:top_k], {"memory_used": len(mem_items)}
-
+def cite_label(c):
+    return f"{c['source']}, {c['loc']}" if c["loc"] else c["source"]
+
+# ========================= INDEX + HYBRID SEARCH =========================
+@st.cache_resource(show_spinner="Loading embedding model...")
+def load_embedder():
+    return SentenceTransformer(EMBED_MODEL_NAME, cache_folder="/tmp/model_cache", device="cpu")
+
+def build_index(chunks, embedder):
+    texts = [light_clean(c["text"]) for c in chunks]
+    emb = embedder.encode(texts, batch_size=32, convert_to_numpy=True,
+                          normalize_embeddings=True, show_progress_bar=False).astype("float32")
+    faiss_index = faiss.IndexFlatIP(emb.shape[1])  # inner product on normalized vectors = cosine
+    faiss_index.add(emb)
+    bm25 = BM25Okapi([tokenize(c["text"]) for c in chunks])
+    return {"chunks": chunks, "faiss": faiss_index, "bm25": bm25}
+
+def hybrid_search(query, index, embedder, top_k, w_sem):
+    """FAISS semantic + BM25, fused with Reciprocal Rank Fusion."""
+    q_vec = embedder.encode([light_clean(query)], convert_to_numpy=True,
+                            normalize_embeddings=True, show_progress_bar=False).astype("float32")
+    n = len(index["chunks"])
+    pool = min(n, top_k * 4)
+
+    scores, ids = index["faiss"].search(q_vec, pool)
+    sem_scores = {int(i): float(s) for i, s in zip(ids[0], scores[0]) if i >= 0}
+    best_sem = float(scores[0][0])
+
+    bm = index["bm25"].get_scores(tokenize(query))
+    bm_rank = np.argsort(-bm)[:pool]
+
+    fused = defaultdict(float)
+    for r, i in enumerate(ids[0]):
+        if i >= 0:
+            fused[int(i)] += w_sem / (RRF_K + r + 1)
+    for r, i in enumerate(bm_rank):
+        if bm[i] > 0:
+            fused[int(i)] += (1 - w_sem) / (RRF_K + r + 1)
+
+    top = sorted(fused, key=fused.get, reverse=True)[:top_k]
+    return top, best_sem, sem_scores, bm
+
+# ========================= GENERATION =========================
 def generate_answer(llm, query, retrieved):
-    context = "\n\n---\n\n".join([f"[Source: {c['source']}]\n{c['text']}" for c in retrieved])
-    prompt = f"""Answer using ONLY the provided context. Cite sources.
+    context = "\n\n---\n\n".join(f"[{cite_label(c)}]\n{c['text']}" for c in retrieved)
+    prompt = f"""You are a document assistant. Answer ONLY from the context below.
+- Answer in the same language as the question.
+- After each fact, cite its source exactly as shown in the context, e.g. [file, p.N].
+- Table rows use " | " between cells; the first line of a table chunk is its header.
+- If the context does not contain the answer, say it was not found in the documents and
+  suggest contacting the document owner. Do not guess.
+
 Context:
 {context}
 
@@ -293,94 +266,99 @@ Answer:"""
     return llm.invoke(prompt).content
 
 # ========================= STREAMLIT APP =========================
-st.set_page_config(page_title="Hybrid Memory Graph RAG", page_icon="🧠", layout="wide")
-os.environ["STREAMLIT_SERVER_FILE_WATCHER_TYPE"] = "none"
-st.title("🧠 Hybrid Memory-Augmented Graph RAG")
+st.set_page_config(page_title="Hybrid RAG Assistant", page_icon="📚", layout="wide")
+st.title("📚 Hybrid RAG Assistant")
+st.caption("FAISS semantic search + BM25, fused with Reciprocal Rank Fusion")
 
-# Session State
-if "memory" not in st.session_state:
-    st.session_state.memory = HybridMemory()
-if "chunks" not in st.session_state:
-    st.session_state.chunks = None
-
-# Sidebar
 with st.sidebar:
-    st.header("Configuration")
-    chunk_method = st.selectbox("Chunking Method", ["Hybrid (Recursive + Semantic)", "Recursive Only"])
-    top_k = st.slider("Top-K Results", 3, 15, DEFAULT_TOP_K)
-    graph_hops = st.slider("Graph Hops", 1, 3, 2)
-    if st.button("Clear Memory"):
-        st.session_state.memory = HybridMemory()
-        st.success("Memory cleared!")
-
-# Load models
-embed_model = load_embed_model()
-reranker = load_reranker()
-nlp = load_spacy()
-lc_embeddings = load_lc_embeddings()
-
-groq_key = get_groq_api_key()
+    st.header("Settings")
+    top_k = st.slider("Chunks sent to the LLM (Top-K)", 3, 10, 5)
+    w_sem = st.slider("Semantic weight (BM25 gets the rest)", 0.0, 1.0, 0.6, 0.1)
+    min_rel = st.slider("Minimum relevance to answer", 0.0, 0.8, 0.25, 0.05,
+                        help="If the best semantic match is below this, the app says 'not found' instead of guessing.")
+
+embedder = load_embedder()
+
+groq_key = None
+try:
+    groq_key = st.secrets.get("GROQ_API_KEY")
+except Exception:
+    pass
+groq_key = groq_key or os.getenv("GROQ_API_KEY")
 if not groq_key:
-    st.error("Groq API key not found!")
+    st.error("GROQ_API_KEY not found. Add it to Streamlit secrets or environment variables.")
     st.stop()
+llm = ChatGroq(temperature=0.1, model_name=LLM_MODEL_NAME, groq_api_key=groq_key)
 
-llm = ChatGroq(temperature=0.2, model_name=LLM_MODEL_NAME, groq_api_key=groq_key)
-
-# File Upload - Now supports PDF, TXT, DOCX
 uploaded_files = st.file_uploader(
-    "Upload Documents (PDF, TXT, DOCX)", 
-    type=["pdf", "txt", "docx"], 
-    accept_multiple_files=True
+    "Upload documents",
+    type=["pdf", "docx", "pptx", "xlsx", "csv", "txt", "md"],
+    accept_multiple_files=True,
 )
 
-if uploaded_files and st.session_state.chunks is None:
-    with st.spinner("Processing documents..."):
-        docs = load_documents(uploaded_files)
-        if not docs:
-            st.error("No readable text found in uploaded files.")
+# Rebuild the index whenever the set of uploaded files changes
+if uploaded_files:
+    signature = tuple(sorted((f.name, f.size) for f in uploaded_files))
+    if st.session_state.get("signature") != signature:
+        all_blocks, report = [], {}
+        with st.spinner("Reading documents..."):
+            for f in uploaded_files:
+                try:
+                    blocks, stats = extract_file(f.name, f.getvalue())
+                except Exception as e:
+                    blocks, stats = [], {"pages": 0, "tables": 0, "notes": [f"Could not read file: {e}"]}
+                all_blocks.extend(blocks)
+                report[f.name] = stats
+        chunks = chunk_blocks(all_blocks)
+        if not chunks:
+            st.error("No readable text found in the uploaded files.")
+            st.session_state.pop("index", None)
+            st.session_state.pop("signature", None)
+            with st.expander("Extraction report", expanded=True):
+                for fname, s in report.items():
+                    for note in s["notes"]:
+                        st.write(f"**{fname}**: {note}")
             st.stop()
-        chunks = chunk_documents(docs, embed_model, chunk_method)
-        bm25, faiss_index, chunk_lookup = build_indexes(chunks, lc_embeddings)
-        graph, entity_to_chunks = build_knowledge_graph(chunks, nlp)
-        
-        st.session_state.chunks = chunks
-        st.session_state.bm25 = bm25
-        st.session_state.faiss_index = faiss_index
-        st.session_state.chunk_lookup = chunk_lookup
-        st.session_state.graph = graph
-        st.session_state.entity_to_chunks = entity_to_chunks
-        st.success(f"Loaded {len(chunks)} chunks from {len(docs)} document(s)")
-
-# Query
-query = st.text_input("Ask a question about your documents:")
-
-if query and st.session_state.chunks:
-    with st.spinner("Retrieving from Documents + Memory..."):
-        retrieved, debug = hybrid_memory_retrieve(
-            query, st.session_state.chunks, st.session_state.bm25,
-            st.session_state.faiss_index, st.session_state.chunk_lookup,
-            st.session_state.graph, st.session_state.entity_to_chunks,
-            reranker, nlp, st.session_state.memory, top_k=top_k, hops=graph_hops
-        )
-    
-    with st.spinner("Generating answer..."):
-        answer = generate_answer(llm, query, retrieved)
-    
-    st.subheader("Answer")
-    st.write(answer)
-
-    if retrieved:
-        summary = f"Q: {query}\nA: {answer[:300]}..."
-        st.session_state.memory.add_memory(summary, memory_type="conversation")
-
-    with st.expander("📊 Memory Stats"):
-        st.json(st.session_state.memory.get_stats())
-
-    with st.expander("🔍 Retrieved Sources"):
-        for i, chunk in enumerate(retrieved, 1):
-            st.markdown(f"**{i}.** {chunk['source']}")
-            st.write(chunk["text"][:500] + "..." if len(chunk["text"]) > 500 else chunk["text"])
+        with st.spinner("Building FAISS + BM25 indexes..."):
+            st.session_state.index = build_index(chunks, embedder)
+        st.session_state.report = report
+        st.session_state.signature = signature
+    st.success(f"Indexed {len(st.session_state.index['chunks'])} chunks from {len(uploaded_files)} file(s)")
+
+    with st.expander("📋 Extraction report"):
+        for fname, s in st.session_state.report.items():
+            st.write(f"**{fname}**: {s['pages']} page(s)/sheet(s), {s['tables']} table(s)")
+            for note in s["notes"]:
+                st.caption(f"⚠️ {note}")
 else:
-    st.info("Upload documents (PDF, TXT, DOCX) and ask a question to begin.")
-
-st.caption("Hybrid Graph RAG + Persistent Memory • Powered by Groq + NetworkX + FAISS")
+    st.session_state.pop("index", None)
+    st.session_state.pop("signature", None)
+
+# Query form: the LLM is only called when the user presses the button
+with st.form("ask"):
+    query = st.text_input("Ask a question about your documents (Arabic or English):")
+    submitted = st.form_submit_button("Ask")
+
+if submitted and query.strip():
+    index = st.session_state.get("index")
+    if not index:
+        st.warning("Upload at least one document first.")
+    else:
+        top, best_sem, sem_scores, bm = hybrid_search(query, index, embedder, top_k, w_sem)
+        retrieved = [index["chunks"][i] for i in top]
+
+        st.subheader("Answer")
+        if not retrieved or best_sem < min_rel:
+            st.info("Not found in the uploaded documents.")
+        else:
+            with st.spinner("Generating answer..."):
+                st.write(generate_answer(llm, query, retrieved))
+
+        with st.expander("🔍 Retrieved chunks"):
+            for rank, i in enumerate(top, 1):
+                c = index["chunks"][i]
+                sem_txt = f"{sem_scores[i]:.2f}" if i in sem_scores else "n/a"
+                st.markdown(f"**{rank}. {cite_label(c)}** ({c['kind']})  ·  semantic {sem_txt}  ·  BM25 {bm[i]:.2f}")
+                st.text(c["text"])
+else:
+    st.info("Upload documents, type a question, and press Ask.")