Spaces:
Sleeping
Sleeping
Download KnowledgeService/parsing.py from dellweb3/workos-office: direct link, hf CLI and curl.
- Browser
- Download file 4.64 kB
-
https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/parsing.py
- Command line
-
hf download hf://spaces/dellweb3/workos-office/KnowledgeService/parsing.py
-
curl -L -o parsing.py https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/parsing.py
4.64 kB
| """Bounded local extraction and deterministic Chinese BM25 retrieval.""" | |
| import io | |
| import re | |
| import unicodedata | |
| import zipfile | |
| from collections import Counter | |
| from defusedxml import ElementTree | |
| from pypdf import PdfReader | |
| from observability import traced | |
| MAX_TEXT = 1_000_000 | |
| MAX_EXPANDED = 20 * 1024 * 1024 | |
| class ParseError(ValueError): | |
| pass | |
| def tokenize(text): | |
| text = unicodedata.normalize("NFKC", text).lower() | |
| terms = [] | |
| for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9_]+", text): | |
| if re.fullmatch(r"[\u3400-\u9fff]+", part): | |
| terms.extend(part[i:i+2] for i in range(len(part)-1)) | |
| if len(part) == 1: | |
| terms.append(part) | |
| else: | |
| terms.append(part) | |
| return terms | |
| def extract_units(filename, content): | |
| suffix = filename.rsplit(".", 1)[-1].lower() | |
| if not content or len(content) > 10 * 1024 * 1024: | |
| raise ParseError("INVALID_FILE_SIZE") | |
| try: | |
| if suffix in ("txt", "md"): | |
| text = content.decode("utf-8-sig") | |
| if "\x00" in text: | |
| raise ParseError("INVALID_TEXT") | |
| units = [("正文", text)] | |
| elif suffix == "docx": | |
| with zipfile.ZipFile(io.BytesIO(content)) as archive: | |
| infos = archive.infolist() | |
| if len(infos) > 2000 or sum(x.file_size for x in infos) > MAX_EXPANDED: | |
| raise ParseError("DOCX_EXPANSION_LIMIT") | |
| if any(x.flag_bits & 1 for x in infos): | |
| raise ParseError("ENCRYPTED_DOCUMENT") | |
| root = ElementTree.fromstring(archive.read("word/document.xml")) | |
| ns = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"} | |
| units = [] | |
| for i, p in enumerate(root.findall(".//w:p", ns), 1): | |
| text = "".join(t.text or "" for t in p.findall(".//w:t", ns)) | |
| units.append((f"段落 {i}", text)) | |
| elif suffix == "pdf": | |
| reader = PdfReader(io.BytesIO(content), strict=True) | |
| if reader.is_encrypted: | |
| raise ParseError("ENCRYPTED_DOCUMENT") | |
| if len(reader.pages) > 200: | |
| raise ParseError("PDF_PAGE_LIMIT") | |
| units = [] | |
| for i, page in enumerate(reader.pages, 1): | |
| stream = page.get_contents() | |
| if stream and len(stream.get_data()) > MAX_EXPANDED: | |
| raise ParseError("PDF_EXPANSION_LIMIT") | |
| text = page.extract_text() or "" | |
| units.append((f"第 {i} 页", text)) | |
| if sum(len(t) for _, t in units) > MAX_TEXT: | |
| raise ParseError("TEXT_SIZE_LIMIT") | |
| else: | |
| raise ParseError("UNSUPPORTED_MEDIA_TYPE") | |
| except ParseError: | |
| raise | |
| except Exception as exc: | |
| raise ParseError("DOCUMENT_PARSE_FAILED") from exc | |
| if sum(len(text) for _, text in units) > MAX_TEXT: | |
| raise ParseError("TEXT_SIZE_LIMIT") | |
| if not any(text.strip() for _, text in units): | |
| raise ParseError("NO_EXTRACTABLE_TEXT") | |
| return units | |
| def extract(filename, content): | |
| units = extract_units(filename, content) | |
| chunks = [] | |
| for locator, text in units: | |
| text = unicodedata.normalize("NFC", text).replace("\r\n", "\n").strip() | |
| if not text: | |
| continue | |
| for offset in range(0, len(text), 700): | |
| part = text[offset:offset+800] | |
| chunks.append((f"{locator} · 字符 {offset+1}–{offset+len(part)}", part, tokenize(part))) | |
| return chunks | |
| def rank(query, rows, top_k): | |
| """Compute statistics only over the already authorized corpus.""" | |
| import math | |
| terms = set(tokenize(query)) | |
| if not terms or not rows: | |
| return [] | |
| counters = [Counter(c.terms) for c, _ in rows] | |
| average = sum(len(c.terms) for c, _ in rows) / len(rows) or 1 | |
| df = {term: sum(term in counts for counts in counters) for term in terms} | |
| scored = [] | |
| for (chunk, document), counts in zip(rows, counters): | |
| score = 0.0 | |
| for term in terms: | |
| tf = counts.get(term, 0) | |
| if tf: | |
| idf = math.log(1 + (len(rows) - df[term] + .5) / (df[term] + .5)) | |
| score += idf * tf * 2.5 / (tf + 1.5 * (.25 + .75 * len(chunk.terms) / average)) | |
| if score > 0: | |
| scored.append((score, chunk, document)) | |
| return sorted(scored, key=lambda x: (-x[0], x[1].id))[:top_k] | |