Update backend/paper_linking.py

backend/paper_linking.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import hashlib
 import re
+import unicodedata
 
 from local_rag.summaries import load_manifest, summary_path
 
@@ -69,7 +70,12 @@ def parse_authors_from_filename(file_name: str) -> list[str]:
     author_part = match.group(1).strip()
     if _ET_AL_RE.search(author_part):
         return [author_part]
-    return [part.strip() for part in re.split(r"\s+and\s+", author_part, flags=re.I) if part.strip()]
+
+    return [
+        part.strip()
+        for part in re.split(r"\s+and\s+", author_part, flags=re.I)
+        if part.strip()
+    ]
 
 
 def parse_authors_from_rp(rp: str) -> str:
@@ -144,47 +150,107 @@ def format_authors(authors: str, *, max_names: int | None = None) -> str:
     """Display string for UI cards. Default shows every parsed author."""
     if not authors:
         return ""
-    parts = [part.strip() for part in authors.split(";") if part.strip()]
+
+    parts = [
+        part.strip()
+        for part in authors.split(";")
+        if part.strip()
+    ]
+
     if max_names is None or len(parts) <= max_names:
         return "; ".join(parts)
+
     return "; ".join(parts[:max_names]) + "; et al."
 
 
+def _normalize_filename(name: str) -> str:
+    """
+    Normalize filenames for reliable matching across Unicode forms.
+
+    Example:
+        Türkiye
+        Türkiye
+
+    Both normalize to the same value.
+
+    NFKC is used to stay consistent with paper_figures.py.
+    """
+    return unicodedata.normalize(
+        "NFKC",
+        (name or "").strip(),
+    ).casefold()
+
+
 def document_id_for_txt_file(txt_file: str) -> str | None:
+    """
+    Resolve a metadata TXT filename to the document_id in manifest.json.
+
+    Matching priority:
+    1. Exact filename match
+    2. Unicode-normalized filename match
+    3. Unicode-normalized match after adding .txt
+    4. Existing summary/hash fallback
+    """
     name = (txt_file or "").strip()
+
     if not name:
         return None
 
     manifest = load_manifest()
     files = manifest.get("files", [])
 
+    # 1. Exact filename match
     for entry in files:
         if entry.get("file_name") == name:
             return entry.get("document_id")
 
-    lowered = name.lower()
+    # 2. Unicode-normalized filename match
+    normalized_name = _normalize_filename(name)
+
     for entry in files:
         file_name = entry.get("file_name", "")
-        if file_name.lower() == lowered:
+
+        if _normalize_filename(file_name) == normalized_name:
             return entry.get("document_id")
 
-    if not name.endswith(".txt"):
+    # 3. If metadata omitted ".txt", try again with the extension
+    if not name.lower().endswith(".txt"):
         with_txt = f"{name}.txt"
+        normalized_with_txt = _normalize_filename(with_txt)
+
         for entry in files:
-            if entry.get("file_name") == with_txt:
+            file_name = entry.get("file_name", "")
+
+            if _normalize_filename(file_name) == normalized_with_txt:
                 return entry.get("document_id")
 
+    # 4. Preserve the original fallback behavior
+    #
+    # Only reached when no corresponding manifest entry was found.
     if summary_path(name) is not None:
-        stem = name[:-4] if name.lower().endswith(".txt") else name
-        return hashlib.sha1(stem.encode("utf-8")).hexdigest()[:16]
+        stem = (
+            name[:-4]
+            if name.lower().endswith(".txt")
+            else name
+        )
+
+        return hashlib.sha1(
+            stem.encode("utf-8")
+        ).hexdigest()[:16]
 
     return None
 
 
 def enrich_metadata_row(row: dict, row_index: int) -> dict:
-    txt_file = (row.get("TXT_file") or row.get("txt_file") or "").strip()
+    txt_file = (
+        row.get("TXT_file")
+        or row.get("txt_file")
+        or ""
+    ).strip()
+
     document_id = document_id_for_txt_file(txt_file)
     authors = resolve_authors(row)
+
     return {
         "metadata_row_index": row_index,
         "title": (row.get("TI") or "").strip(),
@@ -196,4 +262,4 @@ def enrich_metadata_row(row: dict, row_index: int) -> dict:
         "txt_file": txt_file,
         "document_id": document_id,
         "has_chat": document_id is not None,
-    }
+    }