dellweb3's picture
Group original file references and export generated meeting Markdown
2216673 verified
Raw History Blame Contribute Delete
4.64 kB
"""Bounded local extraction and deterministic Chinese BM25 retrieval."""
import io
import re
import unicodedata
import zipfile
from collections import Counter
from defusedxml import ElementTree
from pypdf import PdfReader
from observability import traced
MAX_TEXT = 1_000_000
MAX_EXPANDED = 20 * 1024 * 1024
class ParseError(ValueError):
pass
def tokenize(text):
text = unicodedata.normalize("NFKC", text).lower()
terms = []
for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9_]+", text):
if re.fullmatch(r"[\u3400-\u9fff]+", part):
terms.extend(part[i:i+2] for i in range(len(part)-1))
if len(part) == 1:
terms.append(part)
else:
terms.append(part)
return terms
@traced('Document Parser')
def extract_units(filename, content):
suffix = filename.rsplit(".", 1)[-1].lower()
if not content or len(content) > 10 * 1024 * 1024:
raise ParseError("INVALID_FILE_SIZE")
try:
if suffix in ("txt", "md"):
text = content.decode("utf-8-sig")
if "\x00" in text:
raise ParseError("INVALID_TEXT")
units = [("正文", text)]
elif suffix == "docx":
with zipfile.ZipFile(io.BytesIO(content)) as archive:
infos = archive.infolist()
if len(infos) > 2000 or sum(x.file_size for x in infos) > MAX_EXPANDED:
raise ParseError("DOCX_EXPANSION_LIMIT")
if any(x.flag_bits & 1 for x in infos):
raise ParseError("ENCRYPTED_DOCUMENT")
root = ElementTree.fromstring(archive.read("word/document.xml"))
ns = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"}
units = []
for i, p in enumerate(root.findall(".//w:p", ns), 1):
text = "".join(t.text or "" for t in p.findall(".//w:t", ns))
units.append((f"段落 {i}", text))
elif suffix == "pdf":
reader = PdfReader(io.BytesIO(content), strict=True)
if reader.is_encrypted:
raise ParseError("ENCRYPTED_DOCUMENT")
if len(reader.pages) > 200:
raise ParseError("PDF_PAGE_LIMIT")
units = []
for i, page in enumerate(reader.pages, 1):
stream = page.get_contents()
if stream and len(stream.get_data()) > MAX_EXPANDED:
raise ParseError("PDF_EXPANSION_LIMIT")
text = page.extract_text() or ""
units.append((f"第 {i} 页", text))
if sum(len(t) for _, t in units) > MAX_TEXT:
raise ParseError("TEXT_SIZE_LIMIT")
else:
raise ParseError("UNSUPPORTED_MEDIA_TYPE")
except ParseError:
raise
except Exception as exc:
raise ParseError("DOCUMENT_PARSE_FAILED") from exc
if sum(len(text) for _, text in units) > MAX_TEXT:
raise ParseError("TEXT_SIZE_LIMIT")
if not any(text.strip() for _, text in units):
raise ParseError("NO_EXTRACTABLE_TEXT")
return units
def extract(filename, content):
units = extract_units(filename, content)
chunks = []
for locator, text in units:
text = unicodedata.normalize("NFC", text).replace("\r\n", "\n").strip()
if not text:
continue
for offset in range(0, len(text), 700):
part = text[offset:offset+800]
chunks.append((f"{locator} · 字符 {offset+1}–{offset+len(part)}", part, tokenize(part)))
return chunks
@traced('Search Engine')
def rank(query, rows, top_k):
"""Compute statistics only over the already authorized corpus."""
import math
terms = set(tokenize(query))
if not terms or not rows:
return []
counters = [Counter(c.terms) for c, _ in rows]
average = sum(len(c.terms) for c, _ in rows) / len(rows) or 1
df = {term: sum(term in counts for counts in counters) for term in terms}
scored = []
for (chunk, document), counts in zip(rows, counters):
score = 0.0
for term in terms:
tf = counts.get(term, 0)
if tf:
idf = math.log(1 + (len(rows) - df[term] + .5) / (df[term] + .5))
score += idf * tf * 2.5 / (tf + 1.5 * (.25 + .75 * len(chunk.terms) / average))
if score > 0:
scored.append((score, chunk, document))
return sorted(scored, key=lambda x: (-x[0], x[1].id))[:top_k]