"""Bounded local extraction and deterministic Chinese BM25 retrieval.""" import io import re import unicodedata import zipfile from collections import Counter from defusedxml import ElementTree from pypdf import PdfReader from observability import traced MAX_TEXT = 1_000_000 MAX_EXPANDED = 20 * 1024 * 1024 class ParseError(ValueError): pass def tokenize(text): text = unicodedata.normalize("NFKC", text).lower() terms = [] for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9_]+", text): if re.fullmatch(r"[\u3400-\u9fff]+", part): terms.extend(part[i:i+2] for i in range(len(part)-1)) if len(part) == 1: terms.append(part) else: terms.append(part) return terms @traced('Document Parser') def extract_units(filename, content): suffix = filename.rsplit(".", 1)[-1].lower() if not content or len(content) > 10 * 1024 * 1024: raise ParseError("INVALID_FILE_SIZE") try: if suffix in ("txt", "md"): text = content.decode("utf-8-sig") if "\x00" in text: raise ParseError("INVALID_TEXT") units = [("正文", text)] elif suffix == "docx": with zipfile.ZipFile(io.BytesIO(content)) as archive: infos = archive.infolist() if len(infos) > 2000 or sum(x.file_size for x in infos) > MAX_EXPANDED: raise ParseError("DOCX_EXPANSION_LIMIT") if any(x.flag_bits & 1 for x in infos): raise ParseError("ENCRYPTED_DOCUMENT") root = ElementTree.fromstring(archive.read("word/document.xml")) ns = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"} units = [] for i, p in enumerate(root.findall(".//w:p", ns), 1): text = "".join(t.text or "" for t in p.findall(".//w:t", ns)) units.append((f"段落 {i}", text)) elif suffix == "pdf": reader = PdfReader(io.BytesIO(content), strict=True) if reader.is_encrypted: raise ParseError("ENCRYPTED_DOCUMENT") if len(reader.pages) > 200: raise ParseError("PDF_PAGE_LIMIT") units = [] for i, page in enumerate(reader.pages, 1): stream = page.get_contents() if stream and len(stream.get_data()) > MAX_EXPANDED: raise ParseError("PDF_EXPANSION_LIMIT") text = page.extract_text() or "" units.append((f"第 {i} 页", text)) if sum(len(t) for _, t in units) > MAX_TEXT: raise ParseError("TEXT_SIZE_LIMIT") else: raise ParseError("UNSUPPORTED_MEDIA_TYPE") except ParseError: raise except Exception as exc: raise ParseError("DOCUMENT_PARSE_FAILED") from exc if sum(len(text) for _, text in units) > MAX_TEXT: raise ParseError("TEXT_SIZE_LIMIT") if not any(text.strip() for _, text in units): raise ParseError("NO_EXTRACTABLE_TEXT") return units def extract(filename, content): units = extract_units(filename, content) chunks = [] for locator, text in units: text = unicodedata.normalize("NFC", text).replace("\r\n", "\n").strip() if not text: continue for offset in range(0, len(text), 700): part = text[offset:offset+800] chunks.append((f"{locator} · 字符 {offset+1}–{offset+len(part)}", part, tokenize(part))) return chunks @traced('Search Engine') def rank(query, rows, top_k): """Compute statistics only over the already authorized corpus.""" import math terms = set(tokenize(query)) if not terms or not rows: return [] counters = [Counter(c.terms) for c, _ in rows] average = sum(len(c.terms) for c, _ in rows) / len(rows) or 1 df = {term: sum(term in counts for counts in counters) for term in terms} scored = [] for (chunk, document), counts in zip(rows, counters): score = 0.0 for term in terms: tf = counts.get(term, 0) if tf: idf = math.log(1 + (len(rows) - df[term] + .5) / (df[term] + .5)) score += idf * tf * 2.5 / (tf + 1.5 * (.25 + .75 * len(chunk.terms) / average)) if score > 0: scored.append((score, chunk, document)) return sorted(scored, key=lambda x: (-x[0], x[1].id))[:top_k]