import os import re import numpy as np import gradio as gr import faiss import torch from transformers import AutoTokenizer, AutoModelForQuestionAnswering from sentence_transformers import SentenceTransformer from transformers.utils import logging as hf_logging # Quieter transformers logs (hides the long-sequence warning) hf_logging.set_verbosity_error() # ---------------------------- # Paths (repo-relative, cross-platform) # ---------------------------- BASE_DIR = os.path.dirname(os.path.abspath(__file__)) if "__file__" in globals() else os.getcwd() MODEL_DIR = os.path.join(BASE_DIR, "QA_model") KB_PATH = os.path.join(BASE_DIR, "data", "SalatusSalatin.txt") # ---------------------------- # Devices # ---------------------------- device = "cuda" if torch.cuda.is_available() else "cpu" # ---------------------------- # Load QA model # ---------------------------- qa_tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR) qa_model = AutoModelForQuestionAnswering.from_pretrained(MODEL_DIR).to(device).eval() # ---------------------------- # Embedding model (set a safe max length) # ---------------------------- embedder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device=device) # Ensure the embedder truncates at or below its maximum (prevents 512+ token warnings) embedder.max_seq_length = 256 # typical for MiniLM ST models # ---------------------------- # Chunking helpers # ---------------------------- def split_into_paragraphs(text: str): # split on blank lines, keep only non-empty return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()] def chunks_by_token_count(text: str, tokenizer, max_tokens: int = 220, stride_tokens: int = 40): """ Split `text` into overlapping chunks by *token* count using the provided tokenizer. Ensures each chunk <= max_tokens to avoid long-sequence warnings later. """ # Use the embedder's tokenizer here since retrieval uses these embeddings. enc = tokenizer(text, add_special_tokens=False, return_attention_mask=False, return_token_type_ids=False) ids = enc["input_ids"] chunks = [] start = 0 while start < len(ids): end = min(start + max_tokens, len(ids)) ids_slice = ids[start:end] # decode back to text; skip special tokens just in case chunk_text = tokenizer.decode(ids_slice, skip_special_tokens=True).strip() if chunk_text: chunks.append(chunk_text) if end == len(ids): break start = max(0, end - stride_tokens) return chunks # ---------------------------- # Load KB, make small overlapping chunks (retrieval-friendly) # ---------------------------- with open(KB_PATH, "r", encoding="utf-8") as f: raw_text = f.read() paragraphs = split_into_paragraphs(raw_text) # For retrieval, size ~200–250 tokens tends to work well for MiniLM (we set embedder.max_seq_length=256) RETR_MAX_TOK = 220 RETR_STRIDE = 40 # Use the embedder's tokenizer to guarantee no chunk > its max length embedder_tok = embedder.tokenizer kb_chunks = [] for p in paragraphs: kb_chunks.extend(chunks_by_token_count(p, embedder_tok, max_tokens=RETR_MAX_TOK, stride_tokens=RETR_STRIDE)) # ---------------------------- # Build FAISS index with cosine similarity (normalize + inner product) # ---------------------------- embeddings = embedder.encode(kb_chunks, convert_to_numpy=True, show_progress_bar=True, batch_size=64) embeddings = embeddings.astype("float32") faiss.normalize_L2(embeddings) d = embeddings.shape[1] index = faiss.IndexFlatIP(d) index.add(embeddings) # ---------------------------- # QA over (possibly long) context with sliding window # ---------------------------- def qa_on_context(question: str, context: str, max_length=384, stride=128): """ Run QA model on context using overlapping windows; return (best_answer, best_score). """ inputs = qa_tokenizer( question, context, return_overflowing_tokens=True, truncation=True, # <— CRUCIAL max_length=max_length, # <= 512 for BERT-like models stride=stride, padding="max_length", return_tensors="pt", ) best_answer = "" best_score = -1e30 n = inputs["input_ids"].shape[0] for i in range(n): input_ids = inputs["input_ids"][i].unsqueeze(0).to(device) attention_mask = inputs["attention_mask"][i].unsqueeze(0).to(device) token_type_ids = inputs.get("token_type_ids") if token_type_ids is not None: token_type_ids = token_type_ids[i].unsqueeze(0).to(device) with torch.inference_mode(): outputs = qa_model( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids ) start_scores = outputs.start_logits[0] end_scores = outputs.end_logits[0] # choose best span; optionally enforce end >= start start_idx = int(torch.argmax(start_scores)) end_idx = int(torch.argmax(end_scores)) if end_idx < start_idx: continue score = start_scores[start_idx].item() + end_scores[end_idx].item() answer = qa_tokenizer.decode(input_ids[0][start_idx:end_idx + 1], skip_special_tokens=True).strip() if answer and score > best_score: best_answer, best_score = answer, score return best_answer, best_score # ---------------------------- # End-to-end QA with top-k retrieval + QA re-scoring # ---------------------------- TOP_K = 5 def answer_question(question: str): # 1) Retrieve top-k candidate chunks (cosine sim) q_emb = embedder.encode([question], convert_to_numpy=True).astype("float32") faiss.normalize_L2(q_emb) k = min(TOP_K, len(kb_chunks)) D, I = index.search(q_emb, k=k) cand_idxs = I[0].tolist() # 2) QA each candidate, keep the best by logit score best = {"answer": "", "score": -1e30, "context": ""} for idx in cand_idxs: ctx = kb_chunks[idx] ans, sc = qa_on_context(question, ctx) if sc > best["score"] and ans: best.update({"answer": ans, "score": sc, "context": ctx}) if not best["answer"]: return "Answer: (no good answer found)\n\n(Context: —)" return f"Answer: {best['answer']}\n\n(Context: {best['context']})" examples = [ ["Isi tempat kosong: maka terlalu hairan Patan itu melihat kepandaian Seri Wak Raja pada kuda. Sebermula akan Tun Umar sangat dikasihi _____ ; ianya anak Seri Bija Diraja Datuk Bongkok"], ["Siapakah yang bergajah, Temenggung di kepala gajah, Laksamana atau Seri Bija Diraja di buntut, bentara memikul pedang kerajaan?"], ["Isi tempat kosong: Aria beranakkan Tun Mamad, akan Tun Mamad beranakkan Tun Anjang. Nyaris Lembing Diterbangkan ke Dada\nSekali persetua Sultan Mahmud Syah bermain dengan isteri Tun Biajid, anak _____ ; pada tatkala itu Tun Biajid tiada di rumah, pergi ke Merba, kerana Merba pegangan­nya"], ["Isi tempat kosong: maka Raja Mendaliar pun masuklah mengadap Laksamana, maka segala harta yang dibawanya, semuanya disembahkannya kepada Laksamana. Kata Raja Mendaliar pada Laksamana, \"_____ ini, sahaya berlepas taksir, supaya jangan sahaya dikatakan orang sebicara dengan penghulu sahaya; kerana telah sahaya ketahui bahawa Datuk Bendahara hendak derhaka"], ["Isi tempat kosong: Maka Cau Seri Bangsa pun berperang dengan Raja Sulaiman, sama seekor seorang gajah. Maka kata Cau Seri Bangsa, \"_____ ini olehku, bahawa aku masuk Islamlah.\" Maka dengan takdir Allah Taala, Kota Mahligai pun alahlah, Raja Sulaiman pun mati dibunuh oleh Cau Seri Bangsa, dan rakyat Kota Mahligai pun terhukumlah oleh Cau Seri Bangsa. Maka Cau Seri Bangsa pun masuk Islam."] ] # ---------------------------- # Gradio UI # ---------------------------- iface = gr.Interface( fn=answer_question, inputs="text", outputs="text", title="MALAY-Retrieval-Augmented QA", description="Ask a question; from the Salatus-Salatin.txt the app retrieves the most relevant chunk and extracts the answer. Model trained Using_(bert-base-multilingual-cased)", examples=examples ) if __name__ == "__main__": iface.launch(share = True)