Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -153,7 +153,6 @@ def run_ner(text: str):
|
|
| 153 |
if not word:
|
| 154 |
continue
|
| 155 |
|
| 156 |
-
# ββ Deduplikasi: skip kalau kata ini sudah pernah muncul ββ
|
| 157 |
word_key = word.lower()
|
| 158 |
if word_key in seen_words:
|
| 159 |
continue
|
|
@@ -234,34 +233,27 @@ def run_ner_pdf(pdf_file):
|
|
| 234 |
</div>"""
|
| 235 |
return None, "", warn_html
|
| 236 |
|
| 237 |
-
|
| 238 |
-
# get_text("words") β list of (x0,y0,x1,y1, word, block_no, line_no, word_no)
|
| 239 |
-
pages_words = [] # per halaman: list of (rect, word_lower)
|
| 240 |
full_text_parts = []
|
| 241 |
|
| 242 |
for page in doc:
|
| 243 |
-
|
| 244 |
-
pages_words.append(
|
| 245 |
-
full_text_parts.append(
|
| 246 |
|
| 247 |
full_text = " ".join(full_text_parts)
|
| 248 |
-
|
| 249 |
-
# ββ LANGKAH 2: Masukkan semua teks ke model (sekali jalan) ββ
|
| 250 |
ner_results = ner(full_text.strip())
|
| 251 |
|
| 252 |
if not ner_results:
|
| 253 |
doc.close()
|
| 254 |
return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
|
| 255 |
|
| 256 |
-
# ββ LANGKAH 3: Bangun entity_map dari output model ββ
|
| 257 |
-
# key: kata/frasa (lowercase), value: label_id
|
| 258 |
entity_map: dict[str, str] = {}
|
| 259 |
for ent in ner_results:
|
| 260 |
word = clean_word(ent["word"])
|
| 261 |
if len(word) < 2:
|
| 262 |
continue
|
| 263 |
label_id = get_label_id(ent["entity_group"])
|
| 264 |
-
# Kalau sudah ada, prioritaskan frasa yang lebih panjang
|
| 265 |
if word.lower() not in entity_map:
|
| 266 |
entity_map[word.lower()] = label_id
|
| 267 |
|
|
@@ -269,41 +261,38 @@ def run_ner_pdf(pdf_file):
|
|
| 269 |
doc.close()
|
| 270 |
return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
|
| 271 |
|
| 272 |
-
# Urutkan dari yang terpanjang agar frasa multi-kata dicocokan duluan
|
| 273 |
-
sorted_entities = sorted(entity_map.items(), key=lambda x: len(x[0]), reverse=True)
|
| 274 |
-
|
| 275 |
found_labels: set[str] = set()
|
| 276 |
|
| 277 |
-
# ββ LANGKAH 4 & 5: Cocokan + warnai β per halaman, bukan per entitas ββ
|
| 278 |
for page_idx, page in enumerate(doc):
|
| 279 |
-
|
| 280 |
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
page_text_lower = doc[page_idx].get_text().lower()
|
| 285 |
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
if entity_lower not in page_text_lower:
|
| 289 |
-
continue # skip, tidak ada di halaman ini
|
| 290 |
|
| 291 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
color = LABEL_COLORS.get(label_id, (0.9, 0.9, 0.9))
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
# Simpan ke file temp
|
| 301 |
out_fd, out_path = tempfile.mkstemp(suffix=".pdf")
|
| 302 |
os.close(out_fd)
|
| 303 |
doc.save(out_path)
|
| 304 |
doc.close()
|
| 305 |
|
| 306 |
-
#
|
| 307 |
legend_rows = ""
|
| 308 |
for label_id in sorted(found_labels):
|
| 309 |
label_idn = LABEL_MAP.get(label_id, label_id)
|
|
|
|
| 153 |
if not word:
|
| 154 |
continue
|
| 155 |
|
|
|
|
| 156 |
word_key = word.lower()
|
| 157 |
if word_key in seen_words:
|
| 158 |
continue
|
|
|
|
| 233 |
</div>"""
|
| 234 |
return None, "", warn_html
|
| 235 |
|
| 236 |
+
pages_words = []
|
|
|
|
|
|
|
| 237 |
full_text_parts = []
|
| 238 |
|
| 239 |
for page in doc:
|
| 240 |
+
raw_words = page.get_text("words")
|
| 241 |
+
pages_words.append(raw_words)
|
| 242 |
+
full_text_parts.append(" ".join(w[4] for w in raw_words))
|
| 243 |
|
| 244 |
full_text = " ".join(full_text_parts)
|
|
|
|
|
|
|
| 245 |
ner_results = ner(full_text.strip())
|
| 246 |
|
| 247 |
if not ner_results:
|
| 248 |
doc.close()
|
| 249 |
return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
|
| 250 |
|
|
|
|
|
|
|
| 251 |
entity_map: dict[str, str] = {}
|
| 252 |
for ent in ner_results:
|
| 253 |
word = clean_word(ent["word"])
|
| 254 |
if len(word) < 2:
|
| 255 |
continue
|
| 256 |
label_id = get_label_id(ent["entity_group"])
|
|
|
|
| 257 |
if word.lower() not in entity_map:
|
| 258 |
entity_map[word.lower()] = label_id
|
| 259 |
|
|
|
|
| 261 |
doc.close()
|
| 262 |
return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
|
| 263 |
|
|
|
|
|
|
|
|
|
|
| 264 |
found_labels: set[str] = set()
|
| 265 |
|
|
|
|
| 266 |
for page_idx, page in enumerate(doc):
|
| 267 |
+
raw_words = pages_words[page_idx]
|
| 268 |
|
| 269 |
+
for w in raw_words:
|
| 270 |
+
word_text = w[4]
|
| 271 |
+
word_lower = word_text.lower().strip()
|
|
|
|
| 272 |
|
| 273 |
+
if not word_lower or len(word_lower) < 2:
|
| 274 |
+
continue
|
|
|
|
|
|
|
| 275 |
|
| 276 |
+
# O(1) lookup β langsung cek di dict
|
| 277 |
+
label_id = entity_map.get(word_lower)
|
| 278 |
+
if label_id is None:
|
| 279 |
+
continue
|
| 280 |
+
|
| 281 |
+
# Koordinat sudah ada, langsung buat Rect dan highlight
|
| 282 |
+
rect = fitz.Rect(w[0], w[1], w[2], w[3])
|
| 283 |
color = LABEL_COLORS.get(label_id, (0.9, 0.9, 0.9))
|
| 284 |
+
annot = page.add_highlight_annot(rect)
|
| 285 |
+
annot.set_colors(stroke=list(color))
|
| 286 |
+
annot.update()
|
| 287 |
+
found_labels.add(label_id)
|
| 288 |
+
|
| 289 |
+
# Simpan PDF hasil
|
|
|
|
|
|
|
| 290 |
out_fd, out_path = tempfile.mkstemp(suffix=".pdf")
|
| 291 |
os.close(out_fd)
|
| 292 |
doc.save(out_path)
|
| 293 |
doc.close()
|
| 294 |
|
| 295 |
+
# Legend
|
| 296 |
legend_rows = ""
|
| 297 |
for label_id in sorted(found_labels):
|
| 298 |
label_idn = LABEL_MAP.get(label_id, label_id)
|