RangGaraga commited on
Commit
243d462
Β·
verified Β·
1 Parent(s): ebf1db0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -24
app.py CHANGED
@@ -143,6 +143,8 @@ def run_ner(text: str):
143
 
144
  rows_html = ""
145
  row_num = 1
 
 
146
  for ent in results:
147
  raw_label = ent["entity_group"]
148
  label_id = get_label_id(raw_label)
@@ -151,6 +153,12 @@ def run_ner(text: str):
151
  if not word:
152
  continue
153
 
 
 
 
 
 
 
154
  hex_color = LABEL_HEX.get(label_id, "#e2e8f0")
155
  score = f"{ent['score']:.2%}"
156
  even = row_num % 2 == 0
@@ -171,6 +179,9 @@ def run_ner(text: str):
171
  </tr>"""
172
  row_num += 1
173
 
 
 
 
174
  table_html = f"""
175
  <div style="overflow-x:auto; margin-top:4px; border-radius:12px;
176
  border:1px solid #e2e8f0; box-shadow:0 2px 12px rgba(0,0,0,0.06);">
@@ -193,12 +204,8 @@ def run_ner(text: str):
193
  </div>"""
194
  return table_html
195
 
196
-
197
  # PDF NER
198
  def run_ner_pdf(pdf_file):
199
- """Highlight NER entities in PDF and return (output_path, legend_html, warning_html)."""
200
-
201
- # No file uploaded
202
  if pdf_file is None:
203
  return (
204
  None,
@@ -207,11 +214,9 @@ def run_ner_pdf(pdf_file):
207
  )
208
 
209
  pdf_path = pdf_file if isinstance(pdf_file, str) else pdf_file.name
210
-
211
  doc = fitz.open(pdf_path)
212
  page_count = len(doc)
213
 
214
- # Page limit check
215
  if page_count > MAX_PDF_PAGES:
216
  doc.close()
217
  warn_html = f"""
@@ -224,58 +229,81 @@ def run_ner_pdf(pdf_file):
224
  <p style="margin:4px 0 0; color:#9f1239; font-size:13px;">
225
  File Anda memiliki <strong>{page_count} halaman</strong>.
226
  Maksimal yang diizinkan adalah <strong>{MAX_PDF_PAGES} halaman</strong>.
227
- Silakan unggah PDF yang lebih pendek atau potong halamannya terlebih dahulu.
228
  </p>
229
  </div>
230
  </div>"""
231
  return None, "", warn_html
232
 
233
- # ── Collect all text from pages, run NER ──
234
- full_text = ""
 
 
 
235
  for page in doc:
236
- full_text += page.get_text() + " "
 
 
 
 
237
 
 
238
  ner_results = ner(full_text.strip())
239
 
240
- # Build word β†’ label_id mapping (longest match wins when overlapping)
 
 
 
 
 
241
  entity_map: dict[str, str] = {}
242
  for ent in ner_results:
243
  word = clean_word(ent["word"])
244
  if len(word) < 2:
245
  continue
246
  label_id = get_label_id(ent["entity_group"])
247
- # prefer longer entity spans
248
- if word not in entity_map or len(word) > len(entity_map.get(word, "")):
249
- entity_map[word] = label_id
250
 
251
  if not entity_map:
252
  doc.close()
253
  return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
254
 
255
- # Sort by length descending so longer phrases get highlighted first
256
  sorted_entities = sorted(entity_map.items(), key=lambda x: len(x[0]), reverse=True)
257
 
258
- # Track which labels actually appeared
259
  found_labels: set[str] = set()
260
 
261
- # Highlight each page
262
- for page in doc:
263
- for word, label_id in sorted_entities:
 
 
 
 
 
 
 
 
 
 
 
 
264
  color = LABEL_COLORS.get(label_id, (0.9, 0.9, 0.9))
265
- instances = page.search_for(word, quads=False)
266
  for rect in instances:
267
  annot = page.add_highlight_annot(rect)
268
  annot.set_colors(stroke=list(color))
269
  annot.update()
270
  found_labels.add(label_id)
271
 
272
- # Save to temp file
273
  out_fd, out_path = tempfile.mkstemp(suffix=".pdf")
274
  os.close(out_fd)
275
  doc.save(out_path)
276
  doc.close()
277
 
278
- # Build color legend table
279
  legend_rows = ""
280
  for label_id in sorted(found_labels):
281
  label_idn = LABEL_MAP.get(label_id, label_id)
@@ -318,8 +346,7 @@ def run_ner_pdf(pdf_file):
318
  </div>"""
319
 
320
  return out_path, legend_html, ""
321
-
322
-
323
  # Custom CSS
324
  CUSTOM_CSS = """
325
  @import url('https://fonts.googleapis.com/css2?family=Playfair+Display:wght@700;900&family=DM+Sans:wght@400;500;600&display=swap');
 
143
 
144
  rows_html = ""
145
  row_num = 1
146
+ seen_words = set() # ← tambah ini
147
+
148
  for ent in results:
149
  raw_label = ent["entity_group"]
150
  label_id = get_label_id(raw_label)
 
153
  if not word:
154
  continue
155
 
156
+ # ── Deduplikasi: skip kalau kata ini sudah pernah muncul ──
157
+ word_key = word.lower()
158
+ if word_key in seen_words:
159
+ continue
160
+ seen_words.add(word_key)
161
+
162
  hex_color = LABEL_HEX.get(label_id, "#e2e8f0")
163
  score = f"{ent['score']:.2%}"
164
  even = row_num % 2 == 0
 
179
  </tr>"""
180
  row_num += 1
181
 
182
+ if not rows_html:
183
+ return "<p style='color:#94a3b8; font-style:italic; padding:16px;'>Tidak ada entitas yang ditemukan.</p>"
184
+
185
  table_html = f"""
186
  <div style="overflow-x:auto; margin-top:4px; border-radius:12px;
187
  border:1px solid #e2e8f0; box-shadow:0 2px 12px rgba(0,0,0,0.06);">
 
204
  </div>"""
205
  return table_html
206
 
 
207
  # PDF NER
208
  def run_ner_pdf(pdf_file):
 
 
 
209
  if pdf_file is None:
210
  return (
211
  None,
 
214
  )
215
 
216
  pdf_path = pdf_file if isinstance(pdf_file, str) else pdf_file.name
 
217
  doc = fitz.open(pdf_path)
218
  page_count = len(doc)
219
 
 
220
  if page_count > MAX_PDF_PAGES:
221
  doc.close()
222
  warn_html = f"""
 
229
  <p style="margin:4px 0 0; color:#9f1239; font-size:13px;">
230
  File Anda memiliki <strong>{page_count} halaman</strong>.
231
  Maksimal yang diizinkan adalah <strong>{MAX_PDF_PAGES} halaman</strong>.
 
232
  </p>
233
  </div>
234
  </div>"""
235
  return None, "", warn_html
236
 
237
+ # ── LANGKAH 1: Kumpulkan semua teks per halaman (simpan, jangan hapus) ──
238
+ # get_text("words") β†’ list of (x0,y0,x1,y1, word, block_no, line_no, word_no)
239
+ pages_words = [] # per halaman: list of (rect, word_lower)
240
+ full_text_parts = []
241
+
242
  for page in doc:
243
+ words_on_page = page.get_text("words")
244
+ pages_words.append(words_on_page)
245
+ full_text_parts.append(page.get_text())
246
+
247
+ full_text = " ".join(full_text_parts)
248
 
249
+ # ── LANGKAH 2: Masukkan semua teks ke model (sekali jalan) ──
250
  ner_results = ner(full_text.strip())
251
 
252
+ if not ner_results:
253
+ doc.close()
254
+ return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
255
+
256
+ # ── LANGKAH 3: Bangun entity_map dari output model ──
257
+ # key: kata/frasa (lowercase), value: label_id
258
  entity_map: dict[str, str] = {}
259
  for ent in ner_results:
260
  word = clean_word(ent["word"])
261
  if len(word) < 2:
262
  continue
263
  label_id = get_label_id(ent["entity_group"])
264
+ # Kalau sudah ada, prioritaskan frasa yang lebih panjang
265
+ if word.lower() not in entity_map:
266
+ entity_map[word.lower()] = label_id
267
 
268
  if not entity_map:
269
  doc.close()
270
  return None, "", "<p style='color:#94a3b8; padding:16px;'>Tidak ada entitas yang ditemukan dalam PDF.</p>"
271
 
272
+ # Urutkan dari yang terpanjang agar frasa multi-kata dicocokan duluan
273
  sorted_entities = sorted(entity_map.items(), key=lambda x: len(x[0]), reverse=True)
274
 
 
275
  found_labels: set[str] = set()
276
 
277
+ # ── LANGKAH 4 & 5: Cocokan + warnai β€” per halaman, bukan per entitas ──
278
+ for page_idx, page in enumerate(doc):
279
+ words_on_page = pages_words[page_idx]
280
+
281
+ # Bangun teks halaman lowercase untuk window-matching
282
+ # Kita pakai search_for PyMuPDF tapi HANYA untuk entitas yang
283
+ # memang ADA di halaman ini β€” cek dulu dengan `in page_text`
284
+ page_text_lower = doc[page_idx].get_text().lower()
285
+
286
+ for entity_lower, label_id in sorted_entities:
287
+ # ── Langkah 4: cek keberadaan dulu (O(n) string search, murah) ──
288
+ if entity_lower not in page_text_lower:
289
+ continue # skip, tidak ada di halaman ini
290
+
291
+ # ── Langkah 5: baru panggil search_for (lebih mahal) ──
292
  color = LABEL_COLORS.get(label_id, (0.9, 0.9, 0.9))
293
+ instances = page.search_for(entity_lower, quads=False)
294
  for rect in instances:
295
  annot = page.add_highlight_annot(rect)
296
  annot.set_colors(stroke=list(color))
297
  annot.update()
298
  found_labels.add(label_id)
299
 
300
+ # Simpan ke file temp
301
  out_fd, out_path = tempfile.mkstemp(suffix=".pdf")
302
  os.close(out_fd)
303
  doc.save(out_path)
304
  doc.close()
305
 
306
+ # ── Legend ──
307
  legend_rows = ""
308
  for label_id in sorted(found_labels):
309
  label_idn = LABEL_MAP.get(label_id, label_id)
 
346
  </div>"""
347
 
348
  return out_path, legend_html, ""
349
+
 
350
  # Custom CSS
351
  CUSTOM_CSS = """
352
  @import url('https://fonts.googleapis.com/css2?family=Playfair+Display:wght@700;900&family=DM+Sans:wght@400;500;600&display=swap');