Spaces:
Running on Zero
Running on Zero
Use pypdf for clean token spacing in PDF parser and add definitional sentence priority in grounding engine
Browse files- generate.py +24 -10
- ingest.py +19 -2
generate.py
CHANGED
|
@@ -104,10 +104,22 @@ class OfflineGroundingLLM:
|
|
| 104 |
|
| 105 |
query_stems = {normalize_word(w) for w in query_keywords}
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
# Scan chunks for matching information
|
| 108 |
matching_sentences: List[Tuple[str, DocumentChunk, float]] = []
|
| 109 |
|
| 110 |
-
for chunk in context_chunks:
|
|
|
|
|
|
|
| 111 |
# Check for table chunk
|
| 112 |
if chunk.metadata.get("is_table", False):
|
| 113 |
lines = chunk.text.splitlines()
|
|
@@ -117,8 +129,8 @@ class OfflineGroundingLLM:
|
|
| 117 |
overlap = len(query_stems & line_stems)
|
| 118 |
exact_overlap = len(query_keywords & line_words)
|
| 119 |
if overlap >= 1:
|
| 120 |
-
score = (
|
| 121 |
-
matching_sentences.append((line.strip(), chunk, score
|
| 122 |
continue
|
| 123 |
|
| 124 |
# Split on sentence enders AND newlines (handling notes, bullet points, slides)
|
|
@@ -127,23 +139,25 @@ class OfflineGroundingLLM:
|
|
| 127 |
s_clean = re.sub(r"^[\s*\-•#\d+.)]+", "", s).strip()
|
| 128 |
if len(s_clean.split()) < 3:
|
| 129 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
s_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", s_clean.lower()))
|
| 131 |
s_stems = {normalize_word(w) for w in s_words}
|
| 132 |
overlap = len(query_stems & s_stems)
|
| 133 |
exact_overlap = len(query_keywords & s_words)
|
| 134 |
if overlap >= 1:
|
| 135 |
-
score = (
|
|
|
|
|
|
|
|
|
|
| 136 |
matching_sentences.append((s_clean, chunk, score))
|
| 137 |
|
| 138 |
if not matching_sentences:
|
| 139 |
return "Not found in the document."
|
| 140 |
|
| 141 |
-
# If query has 3+ keywords, prefer sentences with stronger overlap (score >= 1.5)
|
| 142 |
-
if len(query_keywords) >= 3:
|
| 143 |
-
high_matches = [m for m in matching_sentences if m[2] >= 1.5]
|
| 144 |
-
if high_matches:
|
| 145 |
-
matching_sentences = high_matches
|
| 146 |
-
|
| 147 |
# Sort sentences by match score descending
|
| 148 |
matching_sentences.sort(key=lambda x: x[2], reverse=True)
|
| 149 |
|
|
|
|
| 104 |
|
| 105 |
query_stems = {normalize_word(w) for w in query_keywords}
|
| 106 |
|
| 107 |
+
# Helper to detect textbook definitional statements
|
| 108 |
+
def is_definitional_sentence(sentence: str) -> bool:
|
| 109 |
+
s_lower = sentence.lower().strip()
|
| 110 |
+
for kw in query_keywords:
|
| 111 |
+
if re.match(rf"^{kw}\s+(is|are|refers to|means)\b", s_lower):
|
| 112 |
+
return True
|
| 113 |
+
if re.search(rf"\b{kw}\s+(is|are)\s+(the|a|an)\b", s_lower):
|
| 114 |
+
return True
|
| 115 |
+
return False
|
| 116 |
+
|
| 117 |
# Scan chunks for matching information
|
| 118 |
matching_sentences: List[Tuple[str, DocumentChunk, float]] = []
|
| 119 |
|
| 120 |
+
for chunk_idx, chunk in enumerate(context_chunks):
|
| 121 |
+
rank_multiplier = 1.0 / (1.0 + chunk_idx * 0.15)
|
| 122 |
+
|
| 123 |
# Check for table chunk
|
| 124 |
if chunk.metadata.get("is_table", False):
|
| 125 |
lines = chunk.text.splitlines()
|
|
|
|
| 129 |
overlap = len(query_stems & line_stems)
|
| 130 |
exact_overlap = len(query_keywords & line_words)
|
| 131 |
if overlap >= 1:
|
| 132 |
+
score = ((overlap ** 2) * 3.0 + exact_overlap * 2.0) * rank_multiplier
|
| 133 |
+
matching_sentences.append((line.strip(), chunk, score))
|
| 134 |
continue
|
| 135 |
|
| 136 |
# Split on sentence enders AND newlines (handling notes, bullet points, slides)
|
|
|
|
| 139 |
s_clean = re.sub(r"^[\s*\-•#\d+.)]+", "", s).strip()
|
| 140 |
if len(s_clean.split()) < 3:
|
| 141 |
continue
|
| 142 |
+
|
| 143 |
+
# Filter out running page headers / course boilerplate
|
| 144 |
+
if re.search(r"\b(B\.Tech|ICFAI|Tech School|Lecture Notes|All rights reserved)\b", s_clean, re.IGNORECASE):
|
| 145 |
+
continue
|
| 146 |
+
|
| 147 |
s_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", s_clean.lower()))
|
| 148 |
s_stems = {normalize_word(w) for w in s_words}
|
| 149 |
overlap = len(query_stems & s_stems)
|
| 150 |
exact_overlap = len(query_keywords & s_words)
|
| 151 |
if overlap >= 1:
|
| 152 |
+
score = ((overlap ** 2) * 4.0 + (exact_overlap ** 2) * 2.0) * rank_multiplier
|
| 153 |
+
# Definitional boost for core query concepts
|
| 154 |
+
if is_definitional_sentence(s_clean):
|
| 155 |
+
score += 8.0
|
| 156 |
matching_sentences.append((s_clean, chunk, score))
|
| 157 |
|
| 158 |
if not matching_sentences:
|
| 159 |
return "Not found in the document."
|
| 160 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
# Sort sentences by match score descending
|
| 162 |
matching_sentences.sort(key=lambda x: x[2], reverse=True)
|
| 163 |
|
ingest.py
CHANGED
|
@@ -204,6 +204,14 @@ class DocumentIngestionPipeline:
|
|
| 204 |
total_pages = len(pdf.pages)
|
| 205 |
current_section = "Introduction"
|
| 206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
for page_idx, page in enumerate(pdf.pages):
|
| 208 |
page_num = page_idx + 1
|
| 209 |
page_elements = []
|
|
@@ -219,8 +227,17 @@ class DocumentIngestionPipeline:
|
|
| 219 |
except Exception:
|
| 220 |
pass
|
| 221 |
|
| 222 |
-
# 2. Extract text
|
| 223 |
-
raw_text =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
|
| 225 |
# 3. OCR Fallback for scanned/empty pages
|
| 226 |
is_ocr = False
|
|
|
|
| 204 |
total_pages = len(pdf.pages)
|
| 205 |
current_section = "Introduction"
|
| 206 |
|
| 207 |
+
# Pre-load pypdf reader for clean word boundaries and whitespace
|
| 208 |
+
pypdf_reader = None
|
| 209 |
+
if _PYPDF_AVAILABLE:
|
| 210 |
+
try:
|
| 211 |
+
pypdf_reader = pypdf.PdfReader(file_path)
|
| 212 |
+
except Exception:
|
| 213 |
+
pypdf_reader = None
|
| 214 |
+
|
| 215 |
for page_idx, page in enumerate(pdf.pages):
|
| 216 |
page_num = page_idx + 1
|
| 217 |
page_elements = []
|
|
|
|
| 227 |
except Exception:
|
| 228 |
pass
|
| 229 |
|
| 230 |
+
# 2. Extract text: prefer pypdf for clean token separation and proper spacing
|
| 231 |
+
raw_text = ""
|
| 232 |
+
if pypdf_reader is not None and page_idx < len(pypdf_reader.pages):
|
| 233 |
+
try:
|
| 234 |
+
raw_text = pypdf_reader.pages[page_idx].extract_text() or ""
|
| 235 |
+
except Exception:
|
| 236 |
+
raw_text = ""
|
| 237 |
+
|
| 238 |
+
# Fallback to pdfplumber text if pypdf was empty
|
| 239 |
+
if not raw_text.strip():
|
| 240 |
+
raw_text = page.extract_text(layout=False) or ""
|
| 241 |
|
| 242 |
# 3. OCR Fallback for scanned/empty pages
|
| 243 |
is_ocr = False
|