sha22333 commited on
Commit
40ba1fd
·
verified ·
1 Parent(s): ae93d87

Upload 2 files

Browse files
Files changed (2) hide show
  1. pdf_index_enhanced1.pkl +3 -0
  2. rag_pipeline.py +1406 -0
pdf_index_enhanced1.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b79f4c1b524249f1d5ebfa6bbf0e52c4e50b97eb6920b9d3f0a28fff028a6b5
3
+ size 30516213
rag_pipeline.py ADDED
@@ -0,0 +1,1406 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import fitz
3
+ import re
4
+ import pickle
5
+ import numpy as np
6
+ import pandas as pd
7
+ from typing import List, Dict, Tuple, Optional
8
+ from sentence_transformers import SentenceTransformer, CrossEncoder
9
+ from openai import AzureOpenAI
10
+ import logging
11
+ import requests
12
+ from langchain_community.vectorstores import FAISS
13
+ from langchain_community.retrievers import BM25Retriever
14
+ from langchain_core.retrievers import BaseRetriever
15
+ from langchain_core.documents import Document
16
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
17
+ import tiktoken
18
+
19
+ logging.basicConfig(level=logging.INFO)
20
+ logger = logging.getLogger(__name__)
21
+
22
+ class EnsembleRetriever(BaseRetriever):
23
+ """Ensemble retriever that combines multiple retrievers."""
24
+
25
+ retrievers: List[BaseRetriever]
26
+ weights: List[float]
27
+
28
+ def __init__(self, retrievers: List[BaseRetriever], weights: List[float] = None):
29
+ # Calculate weights if not provided
30
+ if weights is None:
31
+ weights = [1.0 / len(retrievers)] * len(retrievers)
32
+
33
+ # Call parent __init__ with the fields as keyword arguments
34
+ super().__init__(retrievers=retrievers, weights=weights)
35
+
36
+ def _get_relevant_documents(self, query: str) -> List[Document]:
37
+ """Get relevant documents from all retrievers."""
38
+ all_docs = []
39
+ doc_scores = {}
40
+
41
+ for retriever, weight in zip(self.retrievers, self.weights):
42
+ # FIXED: Try both invoke() and get_relevant_documents() for compatibility
43
+ try:
44
+ docs = retriever.invoke(query) # New LangChain method
45
+ except AttributeError:
46
+ try:
47
+ docs = retriever.get_relevant_documents(query) # Old method
48
+ except AttributeError:
49
+ logger.warning(f"Retriever {type(retriever)} has no compatible method")
50
+ continue
51
+
52
+ for i, doc in enumerate(docs):
53
+ doc_id = doc.page_content
54
+ score = weight * (1.0 / (i + 1)) # Reciprocal rank fusion
55
+
56
+ if doc_id in doc_scores:
57
+ doc_scores[doc_id]['score'] += score
58
+ else:
59
+ doc_scores[doc_id] = {'doc': doc, 'score': score}
60
+
61
+ # Sort by score and return documents
62
+ sorted_docs = sorted(doc_scores.values(), key=lambda x: x['score'], reverse=True)
63
+ return [item['doc'] for item in sorted_docs]
64
+
65
+ async def _aget_relevant_documents(self, query: str) -> List[Document]:
66
+ """Async version - for now just call sync version."""
67
+ return self._get_relevant_documents(query)
68
+
69
+ def invoke(self, query: str) -> List[Document]:
70
+ """Public invoke method for new LangChain compatibility."""
71
+ return self._get_relevant_documents(query)
72
+
73
+ class RelevanceChecker:
74
+ """
75
+ RelevanceChecker: rerank, threshold, optional contextual compression.
76
+ """
77
+
78
+ def __init__(
79
+ self,
80
+ embedding_model: SentenceTransformer,
81
+ cross_encoder_name: Optional[str] = "cross-encoder/ms-marco-MiniLM-L-6-v2",
82
+ threshold: float = 0.70,
83
+ min_docs: int = 2,
84
+ max_docs: int = 6,
85
+ enable_compression: bool = True,
86
+ compression_top_sentences: int = 3,
87
+ batch_size: int = 32,
88
+ normalize_embeddings: bool = True,
89
+ fallback_to_cosine: bool = True
90
+ ):
91
+ self.embedding_model = embedding_model
92
+ self.cross_encoder = None
93
+ self.cross_encoder_name = cross_encoder_name
94
+
95
+ if cross_encoder_name:
96
+ try:
97
+ self.cross_encoder = CrossEncoder(cross_encoder_name)
98
+ logger.info(f"Loaded CrossEncoder: {cross_encoder_name}")
99
+ except Exception as e:
100
+ logger.warning(f"Failed to load cross-encoder '{cross_encoder_name}': {e}")
101
+ self.cross_encoder = None
102
+
103
+ self.threshold = threshold
104
+ self.min_docs = min_docs
105
+ self.max_docs = max_docs
106
+ self.enable_compression = enable_compression
107
+ self.compression_top_sentences = compression_top_sentences
108
+ self.batch_size = batch_size
109
+ self.normalize_embeddings = normalize_embeddings
110
+ self.fallback_to_cosine = fallback_to_cosine
111
+
112
+ # -------------------------
113
+ # Public API
114
+ # -------------------------
115
+ def filter_documents(
116
+ self,
117
+ question: str,
118
+ docs: List,
119
+ doc_embeddings: Optional[np.ndarray] = None
120
+ ) -> List[Tuple[object, float]]:
121
+
122
+ logger.info(f"Filtering {len(docs)} retrieved chunks for question: {question}")
123
+
124
+ if not docs:
125
+ logger.warning("No documents retrieved.")
126
+ return []
127
+
128
+ # 1) Scoring
129
+ if self.cross_encoder is not None:
130
+ scored = self._score_with_crossencoder(question, docs)
131
+ else:
132
+ scored = self._score_with_cosine(question, docs, doc_embeddings)
133
+
134
+ # Log all scored chunks
135
+ for i, (doc, score) in enumerate(scored):
136
+ logger.info(f"[Score] Rank={i+1} Score={score:.4f} Content={doc.page_content[:150]}...")
137
+
138
+ # 2) Thresholding
139
+ filtered = [(d, s) for d, s in scored if s >= self.threshold]
140
+ logger.info(f"Chunks above threshold {self.threshold}: {len(filtered)}")
141
+
142
+ if len(filtered) < self.min_docs:
143
+ logger.info(f"Below min_docs={self.min_docs}, selecting top {self.min_docs} anyway.")
144
+ filtered = scored[: self.min_docs]
145
+
146
+ filtered = filtered[: self.max_docs]
147
+
148
+ # Log filtered results
149
+ for doc, score in filtered:
150
+ logger.info(f"[Selected] Score={score:.4f} Content={doc.page_content[:150]}...")
151
+
152
+ # 3) Compression
153
+ if self.enable_compression:
154
+ compressed = []
155
+ for doc, score in filtered:
156
+ logger.info(f"Compressing chunk (score={score:.4f}): {doc.page_content[:150]}...")
157
+ compressed_doc = self._compress_document(question, doc, top_k=self.compression_top_sentences)
158
+ logger.info(f"[Compressed Result] {compressed_doc.page_content[:200]}...")
159
+ compressed.append((compressed_doc, score))
160
+ filtered = compressed
161
+
162
+ return filtered
163
+
164
+ # -------------------------
165
+ # Internal scoring helpers
166
+ # -------------------------
167
+ def _score_with_crossencoder(self, question: str, docs: List) -> List[Tuple[object, float]]:
168
+ # Format as tuples (query, passage)
169
+ cross_input = [(question, doc.page_content) for doc in docs]
170
+ try:
171
+ scores = self.cross_encoder.predict(cross_input, batch_size=self.batch_size)
172
+ logger.info("Cross-encoder scoring successful.")
173
+ except Exception as e:
174
+ logger.warning(f"Cross-encoder failed: {e}, falling back to cosine.")
175
+ return self._score_with_cosine(question, docs)
176
+
177
+ scores = self._minmax_normalize(np.array(scores))
178
+ return sorted(list(zip(docs, scores.tolist())), key=lambda x: x[1], reverse=True)
179
+
180
+
181
+ def _score_with_cosine(self, question: str, docs: List, doc_embeddings: Optional[np.ndarray] = None):
182
+ logger.info("Using cosine similarity scoring...")
183
+
184
+ q_emb = self.embedding_model.encode([question], show_progress_bar=False, convert_to_numpy=True)
185
+ if self.normalize_embeddings:
186
+ q_emb = self._l2_normalize(q_emb)
187
+
188
+ if doc_embeddings is None:
189
+ texts = [d.page_content for d in docs]
190
+ doc_embeddings = self.embedding_model.encode(texts, batch_size=self.batch_size,
191
+ show_progress_bar=False, convert_to_numpy=True)
192
+ if self.normalize_embeddings:
193
+ doc_embeddings = self._l2_normalize(doc_embeddings)
194
+
195
+ sims = np.dot(doc_embeddings, q_emb.T).reshape(-1)
196
+ sims = (sims + 1.0) / 2.0 # normalize to 0–1
197
+
198
+ return sorted(list(zip(docs, sims.tolist())), key=lambda x: x[1], reverse=True)
199
+
200
+ # -------------------------
201
+ # Contextual compression
202
+ # -------------------------
203
+ def _compress_document(self, question: str, doc, top_k: int = 3):
204
+ text = doc.page_content
205
+ sentences = self._split_sentences(text)
206
+
207
+ logger.info(f"Splitting into {len(sentences)} sentences for compression.")
208
+
209
+ if not sentences:
210
+ return doc
211
+
212
+ q_emb = self.embedding_model.encode([question], show_progress_bar=False, convert_to_numpy=True)
213
+ sent_embs = self.embedding_model.encode(sentences, batch_size=self.batch_size, show_progress_bar=False, convert_to_numpy=True)
214
+
215
+ if self.normalize_embeddings:
216
+ q_emb = self._l2_normalize(q_emb)
217
+ sent_embs = self._l2_normalize(sent_embs)
218
+
219
+ sims = np.dot(sent_embs, q_emb.T).reshape(-1)
220
+
221
+ # Log top sentences
222
+ idx_scores = list(enumerate(sims))
223
+ idx_scores_sorted = sorted(idx_scores, key=lambda x: x[1], reverse=True)
224
+ for i, (idx, score) in enumerate(idx_scores_sorted[:top_k]):
225
+ logger.info(f"[Compression Sentence #{i+1}] Score={score:.4f} Sentence={sentences[idx][:200]}")
226
+
227
+ top_idx = [i for i, _ in idx_scores_sorted[:top_k]]
228
+ top_idx.sort()
229
+
230
+ compressed_text = " ".join([sentences[i] for i in top_idx]).strip()
231
+
232
+ compressed_doc = type(doc)(
233
+ page_content=compressed_text,
234
+ metadata={**getattr(doc, "metadata", {})}
235
+ )
236
+ return compressed_doc
237
+
238
+ # -------------------------
239
+ # Utilities
240
+ # -------------------------
241
+ @staticmethod
242
+ def _minmax_normalize(arr):
243
+ mn, mx = arr.min(), arr.max()
244
+ return (arr - mn) / (mx - mn + 1e-12)
245
+
246
+ @staticmethod
247
+ def _l2_normalize(x):
248
+ denom = np.linalg.norm(x, axis=1, keepdims=True)
249
+ denom[denom == 0] = 1e-12
250
+ return x / denom
251
+
252
+ @staticmethod
253
+ def _split_sentences(text: str):
254
+ parts = re.split(r'(?<=[\.\?\!])\s+', text)
255
+ return [p.strip() for p in parts if p.strip()]
256
+ class ConversationManager:
257
+ """Manages conversation history with model-aware token limits"""
258
+
259
+ # Context window limits (tokens)
260
+ MODEL_LIMITS = {
261
+ "deepseek": 64000, # DeepSeek-V3
262
+ "azure": 128000, # GPT-4o-mini
263
+ }
264
+
265
+ def __init__(self, llm_type: str, reserve_tokens: int = 8000):
266
+ """
267
+ Args:
268
+ llm_type: "azure" or "deepseek"
269
+ reserve_tokens: Tokens to reserve for system prompt + retrieved docs + response
270
+ """
271
+ self.llm_type = llm_type.lower()
272
+ self.max_context = self.MODEL_LIMITS.get(self.llm_type, 64000)
273
+ self.reserve_tokens = reserve_tokens
274
+ self.available_for_history = self.max_context - self.reserve_tokens
275
+
276
+ # Initialize tokenizer for counting
277
+ try:
278
+ # GPT-4o and DeepSeek use similar tokenization
279
+ self.tokenizer = tiktoken.get_encoding("cl100k_base")
280
+ except:
281
+ logger.warning("Failed to load tiktoken, using fallback estimation")
282
+ self.tokenizer = None
283
+
284
+ self.history = [] # List of {"role": "user/assistant", "content": "..."}
285
+
286
+ logger.info(f"ConversationManager initialized: {self.llm_type}, "
287
+ f"max={self.max_context}, available_for_history={self.available_for_history}")
288
+
289
+ def count_tokens(self, text: str) -> int:
290
+ """Count tokens in text"""
291
+ if self.tokenizer:
292
+ return len(self.tokenizer.encode(text))
293
+ else:
294
+ # Fallback: rough estimation (1 token ≈ 4 characters)
295
+ return len(text) // 4
296
+
297
+ def count_messages_tokens(self, messages: List[Dict[str, str]]) -> int:
298
+ """Count tokens in message list"""
299
+ total = 0
300
+ for msg in messages:
301
+ # Count content tokens
302
+ total += self.count_tokens(msg["content"])
303
+ # Add overhead for message formatting (~4 tokens per message)
304
+ total += 4
305
+ return total
306
+
307
+ def add_exchange(self, user_message: str, assistant_message: str):
308
+ """Add a Q&A pair to history with automatic truncation"""
309
+ # Add new messages
310
+ self.history.append({"role": "user", "content": user_message})
311
+ self.history.append({"role": "assistant", "content": assistant_message})
312
+
313
+ # Truncate if needed
314
+ self._truncate_to_fit()
315
+
316
+ def _truncate_to_fit(self):
317
+ """Remove oldest messages until history fits in available token budget"""
318
+ current_tokens = self.count_messages_tokens(self.history)
319
+
320
+ # Keep removing oldest Q&A pairs until we fit
321
+ while current_tokens > self.available_for_history and len(self.history) > 2:
322
+ # Remove oldest Q&A pair (first 2 messages)
323
+ removed = self.history[:2]
324
+ self.history = self.history[2:]
325
+
326
+ removed_tokens = self.count_messages_tokens(removed)
327
+ current_tokens -= removed_tokens
328
+
329
+ logger.info(f"Truncated conversation: removed {removed_tokens} tokens, "
330
+ f"remaining={current_tokens}/{self.available_for_history}")
331
+
332
+ # Log if we're getting close to limit
333
+ if current_tokens > self.available_for_history * 0.8:
334
+ pairs = len(self.history) // 2
335
+ logger.warning(f"Conversation history at 80% capacity: {current_tokens} tokens, {pairs} pairs")
336
+
337
+ def get_history(self) -> List[Dict[str, str]]:
338
+ """Get current conversation history"""
339
+ return self.history.copy()
340
+
341
+ def get_history_tokens(self) -> int:
342
+ """Get current token count of history"""
343
+ return self.count_messages_tokens(self.history)
344
+
345
+ def clear(self):
346
+ """Clear conversation history"""
347
+ self.history = []
348
+ logger.info("Conversation history cleared")
349
+
350
+ def get_stats(self) -> Dict:
351
+ """Get conversation statistics"""
352
+ pairs = len(self.history) // 2
353
+ tokens = self.count_messages_tokens(self.history)
354
+
355
+ return {
356
+ "total_exchanges": pairs,
357
+ "history_tokens": tokens,
358
+ "available_tokens": self.available_for_history,
359
+ "utilization_percent": round((tokens / self.available_for_history) * 100, 1),
360
+ "model": self.llm_type,
361
+ "max_context": self.max_context
362
+ }
363
+ #sentence-transformers/all-mpnet-base-v2
364
+ class PDFExtractor:
365
+ """Handles PDF extraction with layout preservation"""
366
+
367
+ def __init__(self):
368
+ self.header_footer_margin = 50
369
+ self.min_text_length = 50
370
+ self.heading_font_threshold = 13
371
+
372
+ def extract_pdf(self, pdf_path: str) -> List[Dict]:
373
+ """
374
+ Extract content from PDF with structure preservation
375
+ Returns: List of content blocks with metadata
376
+ """
377
+ try:
378
+ content_blocks = self._extract_with_layout(pdf_path)
379
+
380
+ if not content_blocks:
381
+ logger.warning(f"Layout extraction failed, using fallback for {pdf_path}")
382
+ content_blocks = self._fallback_extraction(pdf_path)
383
+
384
+ # Merge small adjacent blocks
385
+ content_blocks = self._merge_blocks(content_blocks)
386
+
387
+ return content_blocks
388
+
389
+ except Exception as e:
390
+ logger.error(f"PDF extraction failed for {pdf_path}: {e}")
391
+ return []
392
+
393
+ def _extract_with_layout(self, pdf_path: str) -> List[Dict]:
394
+ """Extract text with layout and structure preservation"""
395
+ doc = fitz.open(pdf_path)
396
+ all_content = []
397
+
398
+ for page_num in range(len(doc)):
399
+ page = doc[page_num]
400
+ page_height = page.rect.height
401
+
402
+ # Get text blocks with layout info
403
+ blocks = page.get_text("dict")["blocks"]
404
+
405
+ for block in blocks:
406
+ if "lines" not in block:
407
+ continue
408
+
409
+ bbox = block["bbox"]
410
+
411
+ # Skip headers and footers
412
+ if (bbox[1] < self.header_footer_margin or
413
+ bbox[3] > page_height - self.header_footer_margin):
414
+ continue
415
+
416
+ # Extract text and font information
417
+ text_lines = []
418
+ font_sizes = []
419
+
420
+ for line in block["lines"]:
421
+ line_text = ""
422
+ for span in line["spans"]:
423
+ line_text += span["text"]
424
+ font_sizes.append(span["size"])
425
+
426
+ if line_text.strip():
427
+ text_lines.append(line_text.strip())
428
+
429
+ if not text_lines:
430
+ continue
431
+
432
+ text = " ".join(text_lines)
433
+
434
+ if len(text.strip()) < self.min_text_length:
435
+ continue
436
+
437
+ # Detect headings by font size
438
+ avg_font_size = np.mean(font_sizes) if font_sizes else 11
439
+ content_type = "heading" if avg_font_size > self.heading_font_threshold else "paragraph"
440
+
441
+ all_content.append({
442
+ "text": text,
443
+ "page": page_num + 1,
444
+ "type": content_type,
445
+ "bbox": bbox
446
+ })
447
+
448
+ # Extract tables separately
449
+ tables = self._extract_tables(page, page_num + 1)
450
+ all_content.extend(tables)
451
+
452
+ doc.close()
453
+ return all_content
454
+
455
+ def _extract_tables(self, page, page_num: int) -> List[Dict]:
456
+ """Extract tables from page using PyMuPDF"""
457
+ tables = []
458
+
459
+ try:
460
+ tabs = page.find_tables()
461
+
462
+ for i, table in enumerate(tabs):
463
+ df = table.to_pandas()
464
+
465
+ if df.empty:
466
+ continue
467
+
468
+ table_text = f"Table {i+1}:\n{df.to_string(index=False)}"
469
+
470
+ tables.append({
471
+ "text": table_text,
472
+ "page": page_num,
473
+ "type": "table",
474
+ "bbox": table.bbox
475
+ })
476
+
477
+ except Exception as e:
478
+ logger.debug(f"Table extraction failed on page {page_num}: {e}")
479
+
480
+ return tables
481
+
482
+ def _fallback_extraction(self, pdf_path: str) -> List[Dict]:
483
+ """Simple fallback if advanced extraction fails"""
484
+ doc = fitz.open(pdf_path)
485
+ content = []
486
+
487
+ for page_num in range(len(doc)):
488
+ page = doc[page_num]
489
+ text = page.get_text("text")
490
+
491
+ if text.strip():
492
+ content.append({
493
+ "text": text.strip(),
494
+ "page": page_num + 1,
495
+ "type": "text"
496
+ })
497
+
498
+ doc.close()
499
+ return content
500
+
501
+ def _merge_blocks(self, content_blocks: List[Dict]) -> List[Dict]:
502
+ """Merge small adjacent blocks on same page"""
503
+ if not content_blocks:
504
+ return []
505
+
506
+ merged = []
507
+ current_block = None
508
+
509
+ for block in content_blocks:
510
+ # Always keep tables separate
511
+ if block["type"] == "table":
512
+ if current_block:
513
+ merged.append(current_block)
514
+ current_block = None
515
+ merged.append(block)
516
+ continue
517
+
518
+ if current_block is None:
519
+ current_block = block.copy()
520
+ continue
521
+
522
+ # Merge if same page and combined text not too long
523
+ if (block["page"] == current_block["page"] and
524
+ len(current_block["text"]) + len(block["text"]) < 800):
525
+ current_block["text"] += " " + block["text"]
526
+ else:
527
+ merged.append(current_block)
528
+ current_block = block.copy()
529
+
530
+ if current_block:
531
+ merged.append(current_block)
532
+
533
+ return merged
534
+
535
+ @staticmethod
536
+ def clean_text(text: str) -> str:
537
+ """Clean and normalize extracted text"""
538
+ text = re.sub(r'\s+', ' ', text)
539
+ text = re.sub(r'\b\d{1,3}\s*$', '', text)
540
+ text = re.sub(r'[^\w\s\.\,\:\;\-\(\)\[\]\"\'\%\$\#\@\!\?\/\&\+\=\*]', '', text)
541
+ text = re.sub(r'\s+([.,;:!?)])', r'\1', text)
542
+ text = re.sub(r'([(])\s+', r'\1', text)
543
+ return text.strip()
544
+
545
+
546
+ class RAGPipeline:
547
+ """Main RAG pipeline for PDF question answering"""
548
+ def __init__(
549
+ self,
550
+ pdf_folder: str,
551
+ index_file: str,
552
+ model_params: dict,
553
+ reserve_tokens: int = 8000,
554
+ ):
555
+ # Paths
556
+ self.pdf_folder = pdf_folder
557
+ self.index_file = index_file
558
+
559
+ # Detect LLM type
560
+ llm_type = model_params.get("llm_type", "").lower()
561
+ if llm_type not in ("azure", "deepseek"):
562
+ raise ValueError("model_params['llm_type'] must be 'azure' or 'deepseek'")
563
+
564
+ self.llm_type = llm_type
565
+
566
+ # Initialize conversation manager
567
+ self.conversation_manager = ConversationManager(
568
+ llm_type=self.llm_type,
569
+ reserve_tokens=reserve_tokens
570
+ )
571
+
572
+ #Token swapping
573
+ self.hf_tokens = [model_params["hf_token"]]
574
+ for i in range(1, 11): # Covers hf_backup_token_1 to hf_backup_token_10
575
+ backup_key = f"hf_backup_token_{i}"
576
+ if backup_key in model_params:
577
+ self.hf_tokens.append(model_params[backup_key])
578
+
579
+ # ------------------------------------------------------------------
580
+ # 3. Azure OpenAI
581
+ # ------------------------------------------------------------------
582
+ if self.llm_type == "azure":
583
+ required = ["azure_key", "azure_endpoint", "azure_deployment"]
584
+ missing = [k for k in required if k not in model_params]
585
+ if missing:
586
+ raise ValueError(f"Missing Azure keys in model_params: {missing}")
587
+
588
+ self.llm_client = AzureOpenAI(
589
+ api_key=model_params["azure_key"],
590
+ api_version="2024-02-15-preview",
591
+ azure_endpoint=model_params["azure_endpoint"],
592
+ )
593
+ self.azure_deployment = model_params["azure_deployment"]
594
+
595
+ # ------------------------------------------------------------------
596
+ # 4. DeepSeek (HF Inference API)
597
+ # ------------------------------------------------------------------
598
+ else: # deepseek
599
+ if "hf_token" not in model_params:
600
+ raise ValueError("hf_token is required for DeepSeek")
601
+ self.hf_token = model_params["hf_token"]
602
+ self.deepseek_url = model_params.get("deepseek_url",
603
+ "https://api.deepseek.com/v1/chat/completions")
604
+ self.deepseek_model = model_params.get("deepseek_model", "deepseek-ai/DeepSeek-V3.1:novita")
605
+
606
+ # Models
607
+ self.embedding_model = SentenceTransformer("BAAI/bge-base-en-v1.5")
608
+ self.embedding_model.eval()
609
+ # create relevance checker
610
+ self.relevance_checker = RelevanceChecker(
611
+ embedding_model=self.embedding_model,
612
+ cross_encoder_name="cross-encoder/ms-marco-MiniLM-L-6-v2", # best-effort default
613
+ threshold=0.65,
614
+ min_docs=2,
615
+ max_docs=6,
616
+ enable_compression=False,
617
+ compression_top_sentences=3
618
+ )
619
+
620
+ self.pdf_extractor = PDFExtractor()
621
+
622
+ # Text splitter
623
+ # self.text_splitter = RecursiveCharacterTextSplitter(
624
+ # chunk_size=700,
625
+ # chunk_overlap=150,
626
+ # length_function=len,
627
+ # separators=["\n\n", "\n", ". ", "! ", "? ", "; ", ", ", " ", ""],
628
+ # is_separator_regex=False,
629
+ # )
630
+
631
+ #Simple sliding window chunker.
632
+ # from langchain_text_splitters import CharacterTextSplitter
633
+
634
+ # self.text_splitter = CharacterTextSplitter(
635
+ # chunk_size=512,
636
+ # chunk_overlap=50,
637
+ # separator="\n\n",
638
+ # )
639
+
640
+ #Simple fixed length chunker.
641
+ # from langchain_text_splitters import CharacterTextSplitter
642
+
643
+ # self.text_splitter = CharacterTextSplitter(
644
+ # chunk_size=512,
645
+ # chunk_overlap=0,
646
+ # separator="\n\n",
647
+ # )
648
+
649
+ #Semantic-ish chunker.
650
+ self.text_splitter = SemanticChunker(
651
+ embedding_model=self.embedding_model,
652
+ base_chunk_size=512,
653
+ base_overlap=50,
654
+ sim_threshold=0.85,
655
+ )
656
+ """The Semantica actually worked better than the sliding window chunker."""
657
+ """Fixed length is shape. But it migth not be good to pick up on context."""
658
+ """Sliding window chunker actually made it better. I'll have to tweak more with the chunk_size and chunk_overlap."""
659
+ """When I checked the chunks found in recursive splitter it did have the required texts, but the recursive splitter
660
+ seems to split them at random points leading to a losss of context."""
661
+
662
+ # Storage
663
+ self.documents = []
664
+ self.embeddings = None
665
+ self.faiss_retriever = None
666
+ self.bm25_retriever = None
667
+ self.hybrid_retriever = None
668
+
669
+ def build_index(self, progress_callback=None, status_callback=None):
670
+ """Build index from PDFs in folder"""
671
+ pdf_files = [f for f in os.listdir(self.pdf_folder) if f.lower().endswith(".pdf")]
672
+
673
+ if not pdf_files:
674
+ raise ValueError("No PDF files found in folder")
675
+
676
+ all_documents = []
677
+
678
+ for i, pdf_file in enumerate(pdf_files):
679
+ if status_callback:
680
+ status_callback(f"Processing: {pdf_file} ({i+1}/{len(pdf_files)})")
681
+
682
+ pdf_path = os.path.join(self.pdf_folder, pdf_file)
683
+
684
+ try:
685
+ # Extract content blocks
686
+ content_blocks = self.pdf_extractor.extract_pdf(pdf_path)
687
+
688
+ # Create smart chunks
689
+ documents = self._create_chunks(content_blocks, pdf_file)
690
+ all_documents.extend(documents)
691
+
692
+ logger.info(f"Extracted {len(documents)} chunks from {pdf_file}")
693
+
694
+ except Exception as e:
695
+ logger.error(f"Failed to process {pdf_file}: {e}")
696
+ continue
697
+
698
+ if progress_callback:
699
+ progress_callback((i + 1) / len(pdf_files))
700
+
701
+ if not all_documents:
702
+ raise ValueError("No content extracted from PDFs")
703
+
704
+ if status_callback:
705
+ status_callback(f"Encoding {len(all_documents)} chunks...")
706
+
707
+ # Encode documents
708
+ texts = [doc.page_content for doc in all_documents]
709
+ embeddings = self.embedding_model.encode(
710
+ texts,
711
+ batch_size=32,
712
+ normalize_embeddings=True,
713
+ show_progress_bar=False,
714
+ convert_to_numpy=True
715
+ )
716
+
717
+ # Build retrievers
718
+ self._build_retrievers(all_documents, texts, embeddings)
719
+
720
+ # Save index
721
+ self.documents = all_documents
722
+ self.embeddings = embeddings
723
+ self._save_index()
724
+
725
+ if status_callback:
726
+ status_callback(f"✅ Indexed {len(all_documents)} chunks from {len(pdf_files)} PDFs")
727
+
728
+ return len(all_documents)
729
+
730
+ def _create_chunks(self, content_blocks: List[Dict], pdf_name: str) -> List[Document]:
731
+ """Create smart chunks from content blocks"""
732
+ documents = []
733
+
734
+ for block in content_blocks:
735
+ text = self.pdf_extractor.clean_text(block["text"])
736
+
737
+ if len(text) < 50:
738
+ continue
739
+
740
+ # Keep tables and short content as single chunks
741
+ if block["type"] == "table" or len(text) < 600:
742
+ documents.append(Document(
743
+ page_content=text,
744
+ metadata={
745
+ "source": pdf_name,
746
+ "page": block["page"],
747
+ "type": block["type"]
748
+ }
749
+ ))
750
+ else:
751
+ # Split long content
752
+ chunks = self.text_splitter.split_text(text)
753
+
754
+ for i, chunk in enumerate(chunks):
755
+ documents.append(Document(
756
+ page_content=chunk,
757
+ metadata={
758
+ "source": pdf_name,
759
+ "page": block["page"],
760
+ "type": block["type"],
761
+ "chunk_index": i,
762
+ "total_chunks": len(chunks)
763
+ }
764
+ ))
765
+
766
+ return documents
767
+
768
+ def _build_retrievers(self, documents: List[Document], texts: List[str], embeddings: np.ndarray):
769
+ """Build FAISS and BM25 retrievers"""
770
+ # FAISS retriever
771
+ faiss_index = FAISS.from_embeddings(
772
+ text_embeddings=list(zip(texts, embeddings)),
773
+ embedding=lambda x: self.embedding_model.encode(x, normalize_embeddings=True),
774
+ metadatas=[doc.metadata for doc in documents]
775
+ )
776
+ self.faiss_retriever = faiss_index.as_retriever(search_kwargs={"k": 20})
777
+
778
+ # BM25 retriever
779
+ self.bm25_retriever = BM25Retriever.from_documents(documents)
780
+ self.bm25_retriever.k = 20
781
+
782
+ # Hybrid retriever
783
+ self.hybrid_retriever = EnsembleRetriever(
784
+ retrievers=[self.faiss_retriever, self.bm25_retriever],
785
+ weights=[0.85, 0.15]
786
+ )
787
+
788
+ def _save_index(self):
789
+ """Save index to disk"""
790
+ with open(self.index_file, "wb") as f:
791
+ pickle.dump({
792
+ "documents": self.documents,
793
+ "embeddings": self.embeddings,
794
+ "model": "sentence-transformers/all-mpnet-base-v2"
795
+ }, f)
796
+
797
+ def load_index(self):
798
+ """Load existing index from disk"""
799
+ if not os.path.exists(self.index_file):
800
+ return False
801
+
802
+ try:
803
+ with open(self.index_file, "rb") as f:
804
+ data = pickle.load(f)
805
+
806
+ self.documents = data["documents"]
807
+ self.embeddings = data["embeddings"]
808
+
809
+ # Rebuild retrievers
810
+ texts = [doc.page_content for doc in self.documents]
811
+ self._build_retrievers(self.documents, texts, self.embeddings)
812
+
813
+ logger.info(f"Loaded index with {len(self.documents)} chunks")
814
+ return True
815
+
816
+ except Exception as e:
817
+ logger.error(f"Failed to load index: {e}")
818
+ return False
819
+
820
+ def query(self, question: str, top_k: int = 8) -> Tuple[str, List[Document]]:
821
+
822
+ if self.hybrid_retriever is None:
823
+ raise ValueError("Index not loaded. Call load_index() or build_index() first.")
824
+
825
+ # Retrieve documents
826
+ retrieved_docs = self.hybrid_retriever.invoke(question)
827
+
828
+ if not retrieved_docs:
829
+ answer = "I couldn't find relevant information in the documents."
830
+ return answer, []
831
+
832
+ top_docs = retrieved_docs[:top_k]
833
+ filtered = self.relevance_checker.filter_documents(question, top_docs)
834
+ # filtered is List[(Document, score)]
835
+ filtered_docs = [d for d, s in filtered]
836
+ # Generate answer with history
837
+ answer = self._generate_answer_with_history(question, filtered_docs)
838
+
839
+ # Store this exchange in history
840
+ self.conversation_manager.add_exchange(question, answer)
841
+
842
+ # Log conversation stats
843
+ stats = self.conversation_manager.get_stats()
844
+ logger.info(f"Conversation: {stats['total_exchanges']} exchanges, "
845
+ f"{stats['history_tokens']} tokens ({stats['utilization_percent']}% of available)")
846
+
847
+ return answer, filtered_docs
848
+
849
+ def _generate_answer_with_history(self, question: str, context_docs: List[Document]) -> str:
850
+ """Generate answer using LLM"""
851
+ # Build context
852
+ context_parts = []
853
+ for i, doc in enumerate(context_docs, 1):
854
+ src = doc.metadata.get("source", "Unknown")
855
+ page = doc.metadata.get("page", "?")
856
+ doc_type = doc.metadata.get("type", "text")
857
+
858
+ type_label = f"[{doc_type.upper()}] " if doc_type != "paragraph" else ""
859
+ context_parts.append(
860
+ f"[Source {i}: {src}, Page {page}] {type_label}\n{doc.page_content}\n"
861
+ )
862
+
863
+ context = "\n".join(context_parts)
864
+
865
+ # DEBUG: log context that will be sent to LLM
866
+ logger.info("======== FINAL CONTEXT SENT TO LLM ========")
867
+ logger.info(context[:4000]) # truncate to avoid huge logs
868
+ logger.info("===========================================")
869
+
870
+ system_prompt = """
871
+ You are the **Wetland Conservation and Policy Advisory Assistant (WCPA)** - a specialized AI expert in:
872
+ - Wetland conservation, restoration, and sustainable management
873
+ - Environmental policy analysis and implementation
874
+ - Nature-based solutions and ecosystem services
875
+ - Regulatory frameworks and compliance
876
+
877
+ ## CORE MISSION
878
+ Provide **evidence-based**, **actionable**, and **contextually accurate** guidance to policymakers, environmental planners, researchers, and practitioners regarding wetland protection, restoration, policy implementation, and sustainable resource management.
879
+
880
+ ## RESPONSE FRAMEWORK
881
+
882
+ ### 1. ACCURACY & EVIDENCE STANDARDS
883
+ ✅ **ALWAYS:**
884
+ - Base answers STRICTLY on provided document context
885
+ - Cite specific sources using format: [Source: Document Name, Page X]
886
+ - Distinguish between:
887
+ * **EXPLICIT**: Direct quotes or clear statements from documents
888
+ * **INFERRED**: Logical synthesis across multiple sources (clearly label as "Based on cross-document analysis...")
889
+ - Use conversation history to maintain context and refine answers
890
+
891
+ ❌ **NEVER:**
892
+ - Fabricate data, statistics, policy details, or legal requirements
893
+ - Make up document references or citations
894
+ - Provide information contradicting the source documents
895
+ - Guess when information is unavailable
896
+
897
+ ### 2. WHEN INFORMATION IS MISSING
898
+ If a query cannot be answered from documents:
899
+ 1. State clearly: "The provided documents do not contain specific information about [topic]"
900
+ 2. Explain what related information IS available
901
+ 3. Suggest: "I can provide information on [related topics from docs], or you may need to consult [appropriate authority/resource]"
902
+
903
+ ### 3. STRUCTURED RESPONSE FORMAT
904
+
905
+ **For Specific Questions (e.g., "What is the penalty for wetland violation?"):**
906
+
907
+ **Overview:** (100-150 words)
908
+ - Direct answer to the question
909
+ - Key finding from documents
910
+
911
+ **Evidence from Documents:**
912
+ - Relevant excerpts with citations
913
+ - [Source: Document Name, Page X]: "Exact quote or paraphrased content"
914
+
915
+ **Policy/Legal Context:** (if applicable)
916
+ - Relevant acts, regulations, or frameworks
917
+ - Citation format: [Act Name, Section X, Year]
918
+
919
+ **Practical Implications:**
920
+ - What this means in practice
921
+ - Implementation considerations
922
+
923
+ **Sources:**
924
+ - Full list of referenced documents
925
+
926
+ **Follow-up:** Context-aware question suggesting next logical inquiry
927
+
928
+ ---
929
+
930
+ **For Broad Questions (e.g., "What are wetland restoration strategies?"):**
931
+
932
+ **Overview:** (250-350 words)
933
+ - Comprehensive summary
934
+ - Framework for understanding the topic
935
+
936
+ **Key Strategies/Approaches:**
937
+
938
+ - **Strategy 1 Name**
939
+ [Source: Document Name, Page X]
940
+ - Implementation details
941
+ - Expected outcomes
942
+
943
+ - **Strategy 2 Name**
944
+ [Source: Document Name, Page X]
945
+ - Implementation details
946
+ - Expected outcomes
947
+
948
+ *(One blank line between bullet groups)*
949
+
950
+ **Policy Framework:**
951
+ - Governing policies/acts
952
+ - Regulatory requirements
953
+ - International obligations (e.g., Ramsar Convention)
954
+
955
+ **Implementation Considerations:**
956
+ - Technical requirements
957
+ - Resource needs
958
+ - Stakeholder engagement
959
+ - Monitoring mechanisms
960
+
961
+ **Comparative Data:** (if available)
962
+
963
+ | Approach | Success Rate | Implementation Cost | Timeline |
964
+ |----------|--------------|---------------------|----------|
965
+ | Method A | X% (Source) | $Y (Source) | Z months |
966
+
967
+ **Case Studies:** (if available in documents)
968
+ - Documented examples with outcomes
969
+
970
+ **Sources:**
971
+ - Complete citation list
972
+
973
+ **Follow-up:**
974
+ - Contextual next question
975
+
976
+ ---
977
+
978
+ ### 4. CITATION REQUIREMENTS
979
+
980
+ **Format:** [Source: Document Name, Section/Page X]
981
+
982
+ **Examples:**
983
+ ✅ GOOD: [Source: National Wetland Policy 2015, Section 4.2, Page 12]
984
+ ✅ GOOD: [Source: Ramsar Convention Implementation Report, Page 45-47]
985
+ ❌ BAD: "According to the policy..." (No specific citation)
986
+ ❌ BAD: "Studies show..." (No document reference)
987
+
988
+ **For Multi-Source Claims:**
989
+ - "This approach is supported by multiple sources: [Source A, Page X], [Source B, Page Y], [Source C, Page Z]"
990
+
991
+ ### 5. SPECIAL CONTENT HANDLING
992
+
993
+ **Tables/Quantitative Data:**
994
+ - Reproduce in markdown tables
995
+ - Cite source for each data point
996
+ - Indicate if data is summarized vs. verbatim
997
+
998
+ **Legal/Regulatory Content:**
999
+ - Quote exact section numbers and titles
1000
+ - Cite enactment dates
1001
+ - Note amendments if mentioned
1002
+
1003
+ **Local/Indigenous Knowledge:**
1004
+ - Highlight when referenced in documents
1005
+ - Respect cultural context
1006
+ - Cite community sources if named
1007
+
1008
+ ### 6. TONE & COMMUNICATION
1009
+
1010
+ **Professional but Accessible:**
1011
+ - Technical accuracy without jargon overload
1012
+ - Plain language explanations of complex concepts
1013
+ - Define specialized terms when first used
1014
+
1015
+ **Neutral & Balanced:**
1016
+ - Present all viewpoints found in documents
1017
+ - Note conflicting evidence if exists
1018
+ - Avoid advocacy unless documents explicitly recommend
1019
+
1020
+ **Action-Oriented:**
1021
+ - Emphasize practical implementation steps
1022
+ - Highlight success factors and common challenges
1023
+ - Connect policy to on-ground outcomes
1024
+
1025
+ ### 7. CONTEXT INTEGRATION
1026
+
1027
+ **Use Conversation History:**
1028
+ - Reference previous questions/answers
1029
+ - Build on established context
1030
+ - Avoid repeating information already provided
1031
+ - Make connections: "As discussed earlier regarding [topic]..."
1032
+
1033
+ **Prioritize Recent/Relevant Content:**
1034
+ - Focus on most policy-relevant documents
1035
+ - Highlight recent frameworks over outdated ones (if dates provided)
1036
+ - Note if information may be superseded
1037
+
1038
+ ### 8. INTERCONNECTED THEMES
1039
+
1040
+ Recognize relationships between:
1041
+ - Wetlands ↔ Climate resilience & carbon sequestration
1042
+ - Wetlands ↔ Water security & flood control
1043
+ - Wetlands ↔ Biodiversity & habitat protection
1044
+ - Wetlands ↔ Livelihoods & ecosystem services
1045
+ - Conservation ↔ Sustainable use
1046
+
1047
+ ### 9. RESTRICTIONS
1048
+
1049
+ 🚫 **DO NOT:**
1050
+ - Provide specific financial investment advice
1051
+ - Offer prescriptive legal opinions (defer to legal counsel)
1052
+ - Make political endorsements or criticisms
1053
+ - Recommend specific contractors/vendors
1054
+ - Override document evidence with external knowledge
1055
+
1056
+ ⚠️ **ACKNOWLEDGE:**
1057
+ - Uncertainty when interpreting complex regulations
1058
+ - Conflicting interpretations in documents
1059
+ - Limitations of available data
1060
+
1061
+ ### 10. FOLLOW-UP GUIDANCE
1062
+
1063
+ **End EVERY response with a context-aware question:**
1064
+
1065
+ Examples:
1066
+ - "Would you like detailed implementation steps for [specific strategy]?"
1067
+ - "Should I explain how this aligns with Ramsar reporting obligations?"
1068
+ - "Would you like a comparison of restoration approaches for [wetland type]?"
1069
+ - "Should I provide the regulatory compliance checklist for this activity?"
1070
+ - "Would you like to explore monitoring frameworks for this intervention?"
1071
+
1072
+ ---
1073
+
1074
+ ## QUERY PROCESSING CHECKLIST
1075
+
1076
+ Before responding, verify:
1077
+ - [ ] Retrieved context is relevant to question
1078
+ - [ ] All claims can be cited to specific documents
1079
+ - [ ] Response follows structured format
1080
+ - [ ] Citations use correct format
1081
+ - [ ] Conversation history is integrated
1082
+ - [ ] Follow-up question is contextual and actionable
1083
+ - [ ] Tone is professional and accessible
1084
+ - [ ] No fabricated information included
1085
+
1086
+ ---
1087
+
1088
+ **Remember:** Your authority comes from the documents. When in doubt, cite sources and indicate uncertainty rather than fabricating information. Quality trumps speed - take time to structure clear, well-evidenced responses.
1089
+ """
1090
+
1091
+ user_prompt = f"""CONTEXT FROM DOCUMENTS:
1092
+ {context}
1093
+
1094
+ QUESTION: {question}
1095
+ \n
1096
+ INSTRUCTIONS FOR ANSWERING:
1097
+ - Use the above context **and** the full conversation history.
1098
+ - Always prioritise the **most relevant** and **highest-matching** information.
1099
+ - Follow the Response Format, the System Prompt, and all other detailed instructions strictly.
1100
+ \n
1101
+ ANSWER (with citations):"""
1102
+
1103
+ # Build messages with conversation history
1104
+ messages = [{"role": "system", "content": system_prompt}]
1105
+
1106
+ # Add conversation history
1107
+ history = self.conversation_manager.get_history()
1108
+ if history:
1109
+ messages.extend(history)
1110
+ logger.debug(f"Including {len(history)} history messages "
1111
+ f"({self.conversation_manager.get_history_tokens()} tokens)")
1112
+
1113
+ # Add current query with context
1114
+ messages.append({"role": "user", "content": user_prompt})
1115
+
1116
+ # Log total message tokens
1117
+ if self.conversation_manager.tokenizer:
1118
+ total_input_tokens = self.conversation_manager.count_messages_tokens(messages)
1119
+ logger.info(f"Total input tokens: {total_input_tokens} "
1120
+ f"(limit: {self.conversation_manager.max_context})")
1121
+
1122
+ # --------------------------------------------------------------
1123
+ # CALL THE SELECTED LLM
1124
+ # --------------------------------------------------------------
1125
+ try:
1126
+ if self.llm_type == "azure":
1127
+ resp = self.llm_client.chat.completions.create(
1128
+ model=self.azure_deployment,
1129
+ messages=messages,
1130
+ max_tokens=3500,
1131
+ temperature=0.1,
1132
+ )
1133
+ return resp.choices[0].message.content.strip()
1134
+
1135
+ else: # deepseek
1136
+ headers = {
1137
+ "Authorization": f"Bearer {self.hf_token}",
1138
+ "Content-Type": "application/json"
1139
+ }
1140
+ payload = {
1141
+ "model": self.deepseek_model,
1142
+ "messages": messages,
1143
+ "max_tokens": 3500,
1144
+ "temperature": 0.1
1145
+ }
1146
+
1147
+ logger.info(f"Calling DeepSeek API ({len(messages)} msgs)")
1148
+ for token in self.hf_tokens:
1149
+ headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
1150
+ try:
1151
+ r = requests.post(self.deepseek_url, headers=headers, json=payload, timeout=60)
1152
+ if r.status_code == 200:
1153
+ data = r.json()
1154
+ if data.get("choices"):
1155
+ return data["choices"][0]["message"]["content"].strip()
1156
+ elif r.status_code == 401: # Token expired/invalid
1157
+ continue # Try next token
1158
+ else:
1159
+ logger.error(f"DeepSeek API {r.status_code}: {r.text}")
1160
+ return f"Sorry, model error: {r.status_code} - {r.text}"
1161
+ except Exception as e:
1162
+ logger.error(f"DeepSeek failure with token: {e}")
1163
+ continue # Try next token
1164
+ return "Sorry, all tokens failed."
1165
+
1166
+ except Exception as e:
1167
+ logger.error(f"LLM failure ({self.llm_type}): {e}")
1168
+ return "Sorry, I encountered an error generating the response."
1169
+ def clear_conversation(self):
1170
+ self.conversation_manager.clear()
1171
+
1172
+ def get_conversation_stats(self) -> Dict:
1173
+ return self.conversation_manager.get_stats()
1174
+
1175
+ def get_stats(self) -> Dict:
1176
+ """Get index statistics"""
1177
+ if not self.documents:
1178
+ return {"total_chunks": 0}
1179
+
1180
+ stats = {
1181
+ "total_chunks": len(self.documents),
1182
+ "content_types": {}
1183
+ }
1184
+
1185
+ for doc in self.documents:
1186
+ doc_type = doc.metadata.get("type", "unknown")
1187
+ stats["content_types"][doc_type] = stats["content_types"].get(doc_type, 0) + 1
1188
+
1189
+ return stats
1190
+
1191
+ ##Chunk checker. Only for debugging purposes.
1192
+ def debug_print_chunks_for_source(self, source_name: str, max_chunks: int = 20):
1193
+ """Print all (or first N) chunks for a given PDF source."""
1194
+ matched = [d for d in self.documents if d.metadata.get("source") == source_name]
1195
+ print(f"[DEBUG] Found {len(matched)} chunks for source='{source_name}'")
1196
+ for i, doc in enumerate(matched[:max_chunks], 1):
1197
+ print(f"\n--- Chunk {i} ---")
1198
+ print("metadata:", doc.metadata)
1199
+ preview = doc.page_content[:800].replace("\n", " ")
1200
+ print("text :", preview, "...")
1201
+
1202
+ class SemanticChunker:
1203
+ """
1204
+ Very simple semantic-ish chunker:
1205
+ 1) First split into small fixed chunks.
1206
+ 2) Then merge adjacent chunks whose embeddings are very similar.
1207
+ """
1208
+
1209
+ def __init__(
1210
+ self,
1211
+ embedding_model: SentenceTransformer,
1212
+ base_chunk_size: int = 512,
1213
+ base_overlap: int = 50,
1214
+ sim_threshold: float = 0.85,
1215
+ ):
1216
+ from langchain_text_splitters import CharacterTextSplitter
1217
+
1218
+ self.embedding_model = embedding_model
1219
+ self.base_splitter = CharacterTextSplitter(
1220
+ chunk_size=base_chunk_size,
1221
+ chunk_overlap=base_overlap,
1222
+ separator="\n\n",
1223
+ )
1224
+ self.sim_threshold = sim_threshold
1225
+
1226
+ def split_text(self, text: str) -> list[str]:
1227
+ # 1) base split
1228
+ mini_chunks = self.base_splitter.split_text(text)
1229
+ if len(mini_chunks) <= 1:
1230
+ return mini_chunks
1231
+
1232
+ # 2) embed all mini-chunks
1233
+ embs = self.embedding_model.encode(
1234
+ mini_chunks, show_progress_bar=False, convert_to_numpy=True, normalize_embeddings=True
1235
+ )
1236
+
1237
+ # 3) merge adjacent chunks with high cosine similarity
1238
+ merged_chunks: list[str] = []
1239
+ current = mini_chunks[0]
1240
+ current_emb = embs[0]
1241
+
1242
+ for i in range(1, len(mini_chunks)):
1243
+ sim = float(np.dot(current_emb, embs[i]))
1244
+ if sim >= self.sim_threshold:
1245
+ # same topic: merge
1246
+ current = current + " " + mini_chunks[i]
1247
+ # update embedding as average of both (approx)
1248
+ current_emb = (current_emb + embs[i]) / 2.0
1249
+ else:
1250
+ merged_chunks.append(current)
1251
+ current = mini_chunks[i]
1252
+ current_emb = embs[i]
1253
+
1254
+ merged_chunks.append(current)
1255
+ return merged_chunks
1256
+
1257
+ if __name__ == "__main__":
1258
+ """
1259
+ Manual index builder / inspector.
1260
+ Run from terminal:
1261
+ python rag_pipeline.py
1262
+ """
1263
+
1264
+ PDF_FOLDER = r"C:\Users\A.Kumarasiri\OneDrive - CGIAR\WETLAND CHATBOT DOCUMENT\ALL"
1265
+ INDEX_FILE = "pdf_index_enhanced1.pkl"
1266
+
1267
+ # Minimal model params – just enough to construct RAGPipeline.
1268
+ # We won't call .query() here, so the tokens/URL don't actually get used.
1269
+ model_params = {
1270
+ "llm_type": "deepseek",
1271
+ "hf_token": f'st.secrets["hf_backup_token_2"]',
1272
+ "deepseek_url": "https://router.huggingface.co/v1/chat/completions",
1273
+ "deepseek_model": "deepseek-ai/DeepSeek-R1:novita",
1274
+ }
1275
+
1276
+ print("[MAIN] Initializing RAGPipeline...")
1277
+ pipeline = RAGPipeline(
1278
+ pdf_folder=PDF_FOLDER,
1279
+ index_file=INDEX_FILE,
1280
+ model_params=model_params,
1281
+ )
1282
+
1283
+ # 1) Try to load existing index
1284
+ if pipeline.load_index():
1285
+ print("[MAIN] Existing index loaded.")
1286
+ else:
1287
+ print("[MAIN] No index found or failed to load. Building a new one...")
1288
+ try:
1289
+ total_chunks = pipeline.build_index()
1290
+ print(f"[MAIN] Index built successfully. Total chunks: {total_chunks}")
1291
+ except Exception as e:
1292
+ print("[MAIN] Index build failed:", e)
1293
+ raise
1294
+
1295
+ """
1296
+ View Chunks Script
1297
+ Displays chunks from the RAG pipeline index
1298
+ """
1299
+
1300
+ import pickle
1301
+
1302
+ # ============================================================================
1303
+ # CONFIGURATION
1304
+ # ============================================================================
1305
+ INDEX_FILE = "pdf_index_enhanced.pkl"
1306
+ TARGET_PDF = "National Environmental Policy and Strategies (2003).pdf" # Change this to view different PDF
1307
+ MAX_CHUNKS_TO_SHOW = 20
1308
+
1309
+ # ============================================================================
1310
+ # FUNCTIONS
1311
+ # ============================================================================
1312
+ def load_index(index_file: str):
1313
+ """Load the pickled index and return documents"""
1314
+ print(f"Loading index from: {index_file}")
1315
+ with open(index_file, "rb") as f:
1316
+ data = pickle.load(f)
1317
+ print(f"✅ Loaded {len(data['documents'])} documents\n")
1318
+ return data["documents"]
1319
+
1320
+ def show_sample_chunks(documents, max_samples: int = 20):
1321
+ """Show sample chunks from different PDFs (max 1 per source PDF)"""
1322
+ if not documents:
1323
+ print("[MAIN] No documents in pipeline.documents after load/build.")
1324
+ return
1325
+
1326
+ print(f"\n[MAIN] Showing up to {max_samples} sample chunks from different PDFs")
1327
+ print(f" (total chunks: {len(documents)})")
1328
+
1329
+ seen_sources = set()
1330
+ shown = 0
1331
+
1332
+ for doc in documents:
1333
+ src = doc.metadata.get("source", "Unknown")
1334
+ if src in seen_sources:
1335
+ continue
1336
+ seen_sources.add(src)
1337
+ shown += 1
1338
+
1339
+ print(f"\n--- Sample {shown} ---")
1340
+ print("source :", src)
1341
+ print("metadata:", doc.metadata)
1342
+ preview = doc.page_content[:400].replace("\n", " ")
1343
+ print("text :", preview, "...")
1344
+
1345
+ if shown >= max_samples:
1346
+ break
1347
+
1348
+ def debug_print_chunks_for_source(documents, source_name: str, max_chunks: int = 20):
1349
+ """Print all (or first N) chunks for a given PDF source"""
1350
+ matched = [d for d in documents if d.metadata.get("source") == source_name]
1351
+
1352
+ print(f"\n{'='*80}")
1353
+ print(f"[MAIN] Debug: chunks for {source_name}")
1354
+ print(f"{'='*80}")
1355
+ print(f"[DEBUG] Found {len(matched)} chunks for source='{source_name}'")
1356
+
1357
+ if not matched:
1358
+ print(f"❌ No chunks found for '{source_name}'")
1359
+ print("\nAvailable PDFs:")
1360
+ unique_sources = sorted(set(d.metadata.get("source", "Unknown") for d in documents))
1361
+ for i, src in enumerate(unique_sources[:15], 1):
1362
+ print(f" {i:2d}. {src}")
1363
+ if len(unique_sources) > 15:
1364
+ print(f" ... and {len(unique_sources) - 15} more")
1365
+ return
1366
+
1367
+ print(f"Showing first {min(max_chunks, len(matched))} chunks:\n")
1368
+
1369
+ for i, doc in enumerate(matched[:max_chunks], 1):
1370
+ print(f"--- Chunk {i} ---")
1371
+ print("source :", doc.metadata.get("source"))
1372
+ print("metadata:", doc.metadata)
1373
+ preview = doc.page_content[:800].replace("\n", " ")
1374
+ print("text :", preview, "...")
1375
+ print()
1376
+
1377
+ #
1378
+
1379
+ # 2) Show a spread of sample chunks for inspection (max 1 per source PDF)
1380
+ # if pipeline.documents:
1381
+ # print(f"\n[MAIN] Showing up to 20 sample chunks from different PDFs "
1382
+ # f"(total chunks: {len(pipeline.documents)}):")
1383
+
1384
+ # seen_sources = set()
1385
+ # shown = 0
1386
+ # for doc in pipeline.documents:
1387
+ # src = doc.metadata.get("source", "Unknown")
1388
+ # if src in seen_sources:
1389
+ # continue
1390
+ # seen_sources.add(src)
1391
+ # shown += 1
1392
+
1393
+ # print(f"\n--- Sample {shown} ---")
1394
+ # print("source :", src)
1395
+ # print("metadata:", doc.metadata)
1396
+ # preview = doc.page_content[:400].replace("\n", " ")
1397
+ # print("text :", preview, "...")
1398
+
1399
+ # if shown >= 20:
1400
+ # break
1401
+ # else:
1402
+ # print("[MAIN] No documents in pipeline.documents after load/build.")
1403
+
1404
+ # print("\n[MAIN] Debug: chunks for Rao-2018-Power_from_agro-waste-Business_Model_6.pdf")
1405
+ # pipeline.debug_print_chunks_for_source("Rao-2018-Power_from_agro-waste-Business_Model_6.pdf",
1406
+ # max_chunks=20)