Varun2007 commited on
Commit
42a9da6
·
1 Parent(s): 74f3acd

Use pypdf for clean token spacing in PDF parser and add definitional sentence priority in grounding engine

Browse files
Files changed (2) hide show
  1. generate.py +24 -10
  2. ingest.py +19 -2
generate.py CHANGED
@@ -104,10 +104,22 @@ class OfflineGroundingLLM:
104
 
105
  query_stems = {normalize_word(w) for w in query_keywords}
106
 
 
 
 
 
 
 
 
 
 
 
107
  # Scan chunks for matching information
108
  matching_sentences: List[Tuple[str, DocumentChunk, float]] = []
109
 
110
- for chunk in context_chunks:
 
 
111
  # Check for table chunk
112
  if chunk.metadata.get("is_table", False):
113
  lines = chunk.text.splitlines()
@@ -117,8 +129,8 @@ class OfflineGroundingLLM:
117
  overlap = len(query_stems & line_stems)
118
  exact_overlap = len(query_keywords & line_words)
119
  if overlap >= 1:
120
- score = (exact_overlap * 2.0 + overlap) / max(1, len(query_keywords))
121
- matching_sentences.append((line.strip(), chunk, score * 1.5))
122
  continue
123
 
124
  # Split on sentence enders AND newlines (handling notes, bullet points, slides)
@@ -127,23 +139,25 @@ class OfflineGroundingLLM:
127
  s_clean = re.sub(r"^[\s*\-•#\d+.)]+", "", s).strip()
128
  if len(s_clean.split()) < 3:
129
  continue
 
 
 
 
 
130
  s_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", s_clean.lower()))
131
  s_stems = {normalize_word(w) for w in s_words}
132
  overlap = len(query_stems & s_stems)
133
  exact_overlap = len(query_keywords & s_words)
134
  if overlap >= 1:
135
- score = (exact_overlap * 2.0 + overlap) / max(1, len(query_keywords))
 
 
 
136
  matching_sentences.append((s_clean, chunk, score))
137
 
138
  if not matching_sentences:
139
  return "Not found in the document."
140
 
141
- # If query has 3+ keywords, prefer sentences with stronger overlap (score >= 1.5)
142
- if len(query_keywords) >= 3:
143
- high_matches = [m for m in matching_sentences if m[2] >= 1.5]
144
- if high_matches:
145
- matching_sentences = high_matches
146
-
147
  # Sort sentences by match score descending
148
  matching_sentences.sort(key=lambda x: x[2], reverse=True)
149
 
 
104
 
105
  query_stems = {normalize_word(w) for w in query_keywords}
106
 
107
+ # Helper to detect textbook definitional statements
108
+ def is_definitional_sentence(sentence: str) -> bool:
109
+ s_lower = sentence.lower().strip()
110
+ for kw in query_keywords:
111
+ if re.match(rf"^{kw}\s+(is|are|refers to|means)\b", s_lower):
112
+ return True
113
+ if re.search(rf"\b{kw}\s+(is|are)\s+(the|a|an)\b", s_lower):
114
+ return True
115
+ return False
116
+
117
  # Scan chunks for matching information
118
  matching_sentences: List[Tuple[str, DocumentChunk, float]] = []
119
 
120
+ for chunk_idx, chunk in enumerate(context_chunks):
121
+ rank_multiplier = 1.0 / (1.0 + chunk_idx * 0.15)
122
+
123
  # Check for table chunk
124
  if chunk.metadata.get("is_table", False):
125
  lines = chunk.text.splitlines()
 
129
  overlap = len(query_stems & line_stems)
130
  exact_overlap = len(query_keywords & line_words)
131
  if overlap >= 1:
132
+ score = ((overlap ** 2) * 3.0 + exact_overlap * 2.0) * rank_multiplier
133
+ matching_sentences.append((line.strip(), chunk, score))
134
  continue
135
 
136
  # Split on sentence enders AND newlines (handling notes, bullet points, slides)
 
139
  s_clean = re.sub(r"^[\s*\-•#\d+.)]+", "", s).strip()
140
  if len(s_clean.split()) < 3:
141
  continue
142
+
143
+ # Filter out running page headers / course boilerplate
144
+ if re.search(r"\b(B\.Tech|ICFAI|Tech School|Lecture Notes|All rights reserved)\b", s_clean, re.IGNORECASE):
145
+ continue
146
+
147
  s_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", s_clean.lower()))
148
  s_stems = {normalize_word(w) for w in s_words}
149
  overlap = len(query_stems & s_stems)
150
  exact_overlap = len(query_keywords & s_words)
151
  if overlap >= 1:
152
+ score = ((overlap ** 2) * 4.0 + (exact_overlap ** 2) * 2.0) * rank_multiplier
153
+ # Definitional boost for core query concepts
154
+ if is_definitional_sentence(s_clean):
155
+ score += 8.0
156
  matching_sentences.append((s_clean, chunk, score))
157
 
158
  if not matching_sentences:
159
  return "Not found in the document."
160
 
 
 
 
 
 
 
161
  # Sort sentences by match score descending
162
  matching_sentences.sort(key=lambda x: x[2], reverse=True)
163
 
ingest.py CHANGED
@@ -204,6 +204,14 @@ class DocumentIngestionPipeline:
204
  total_pages = len(pdf.pages)
205
  current_section = "Introduction"
206
 
 
 
 
 
 
 
 
 
207
  for page_idx, page in enumerate(pdf.pages):
208
  page_num = page_idx + 1
209
  page_elements = []
@@ -219,8 +227,17 @@ class DocumentIngestionPipeline:
219
  except Exception:
220
  pass
221
 
222
- # 2. Extract text with layout awareness
223
- raw_text = page.extract_text(layout=False) or ""
 
 
 
 
 
 
 
 
 
224
 
225
  # 3. OCR Fallback for scanned/empty pages
226
  is_ocr = False
 
204
  total_pages = len(pdf.pages)
205
  current_section = "Introduction"
206
 
207
+ # Pre-load pypdf reader for clean word boundaries and whitespace
208
+ pypdf_reader = None
209
+ if _PYPDF_AVAILABLE:
210
+ try:
211
+ pypdf_reader = pypdf.PdfReader(file_path)
212
+ except Exception:
213
+ pypdf_reader = None
214
+
215
  for page_idx, page in enumerate(pdf.pages):
216
  page_num = page_idx + 1
217
  page_elements = []
 
227
  except Exception:
228
  pass
229
 
230
+ # 2. Extract text: prefer pypdf for clean token separation and proper spacing
231
+ raw_text = ""
232
+ if pypdf_reader is not None and page_idx < len(pypdf_reader.pages):
233
+ try:
234
+ raw_text = pypdf_reader.pages[page_idx].extract_text() or ""
235
+ except Exception:
236
+ raw_text = ""
237
+
238
+ # Fallback to pdfplumber text if pypdf was empty
239
+ if not raw_text.strip():
240
+ raw_text = page.extract_text(layout=False) or ""
241
 
242
  # 3. OCR Fallback for scanned/empty pages
243
  is_ocr = False