Spaces:
Running
Running
Shoufa Zhang
Deploy DocMorph & SeekMorph interactive Studio and 5-layer production benchmark
66227f0 Download tests/test_docmorph.py from zhangshoufa/DocMorph: direct link, hf CLI and curl.
- Browser
- Download file 6.21 kB
-
https://huggingface.co/spaces/zhangshoufa/DocMorph/resolve/main/tests/test_docmorph.py
- Command line
-
hf download hf://spaces/zhangshoufa/DocMorph/tests/test_docmorph.py
-
curl -L -o test_docmorph.py https://huggingface.co/spaces/zhangshoufa/DocMorph/resolve/main/tests/test_docmorph.py
6.21 kB
| """End-to-end tests for DocMorph parser, table header preservation, hybrid index, agent toolkit, and CLI.""" | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from docmorph import ( | |
| DocMorphAgentToolkit, | |
| DocumentMorpher, | |
| HybridIndex, | |
| csv_to_markdown_table, | |
| html_to_markdown, | |
| normalize_parallel_tool_calls, | |
| ) | |
| from docmorph.cli import main as cli_main | |
| def test_html_to_markdown_converts_headings_tables_and_code(): | |
| html = """ | |
| <html> | |
| <head><script>console.log("ignore")</script></head> | |
| <body> | |
| <h1>Deployment Guide</h1> | |
| <p>Run the server with speculative decoding enabled.</p> | |
| <table> | |
| <tr><th>Hardware</th><th>Acceptance</th></tr> | |
| <tr><td>MI308X</td><td>3.12</td></tr> | |
| </table> | |
| <pre><code>vllm serve deepseek-v4</code></pre> | |
| </body> | |
| </html> | |
| """ | |
| md = html_to_markdown(html) | |
| assert "# Deployment Guide" in md | |
| assert "| Hardware | Acceptance |" in md | |
| assert "| MI308X | 3.12 |" in md | |
| assert "vllm serve deepseek-v4" in md | |
| assert "console.log" not in md | |
| def test_large_table_splitting_repeats_header_in_every_chunk(): | |
| rows = ["metric,q1,q2"] + [f"row_{i},100_{i},200_{i}" for i in range(25)] | |
| csv_text = "\n".join(rows) | |
| morpher = DocumentMorpher(max_chars=180) | |
| chunks = morpher.morph_text("financials.csv", csv_text, fmt="csv") | |
| assert len(chunks) >= 3 | |
| for chunk in chunks: | |
| assert chunk.kind == "table" | |
| lines = chunk.content.splitlines() | |
| assert lines[0] == "| metric | q1 | q2 |" | |
| assert lines[1] == "| --- | --- | --- |" | |
| def test_hybrid_index_multilingual_bm25_and_dense_rrf_with_persistence(tmp_path: Path): | |
| morpher = DocumentMorpher(max_chars=400) | |
| md = ( | |
| "# 架构设计\n\n" | |
| "## 投机采样加速\n\n" | |
| "DeepSeek-V4 使用 DSpark 投机采样,在 8 卡 MI308X 上实现 3.1 倍吞吐提升。\n\n" | |
| "## 混合检索模块\n\n" | |
| "结合 BM25 词法匹配与稠密向量余弦相似度,通过 RRF 融合排序。\n" | |
| ) | |
| chunks = morpher.morph_text("design.md", md, namespace="prod") | |
| assert len(chunks) == 2 | |
| index = HybridIndex() | |
| index.add(chunks[0], embedding=[0.9, 0.1]) | |
| index.add(chunks[1], embedding=[0.1, 0.9]) | |
| index_path = tmp_path / "kb.jsonl" | |
| index.save(index_path) | |
| # Append a corrupted partial line to verify fault-tolerant loading | |
| with index_path.open("a", encoding="utf-8") as f: | |
| f.write("{corrupted_json_line\n") | |
| reloaded = HybridIndex.load(index_path) | |
| assert len(reloaded) == 2 | |
| hits = reloaded.search("投机采样 MI308X", top_k=1, namespace="prod", query_embedding=[0.95, 0.05]) | |
| assert len(hits) == 1 | |
| assert hits[0].chunk.section_path == ("架构设计", "投机采样加速") | |
| assert "3.1 倍" in hits[0].expanded_content | |
| assert reloaded.remove_document("design.md", namespace="prod") == 2 | |
| assert len(reloaded) == 0 | |
| assert reloaded.search("投机采样 MI308X", top_k=1, namespace="prod") == [] | |
| def test_context_window_expansion_within_same_section(): | |
| morpher = DocumentMorpher(max_chars=90) | |
| md = ( | |
| "# Section A\n\n" | |
| "Paragraph one introduces the system configuration parameters and baseline setup.\n\n" | |
| "Paragraph two explains the exact token acceptance threshold and fallback behavior.\n\n" | |
| "Paragraph three summarizes the final benchmark numbers on production traffic.\n" | |
| ) | |
| chunks = morpher.morph_text("guide.md", md) | |
| assert len(chunks) == 3 | |
| index = HybridIndex() | |
| index.add_many(chunks) | |
| hits = index.search("acceptance threshold", top_k=1, expand_window=1) | |
| assert len(hits) == 1 | |
| assert "Paragraph one" in hits[0].expanded_content | |
| assert "Paragraph two" in hits[0].expanded_content | |
| assert "Paragraph three" in hits[0].expanded_content | |
| def test_agent_toolkit_and_parallel_tool_call_normalization(): | |
| morpher = DocumentMorpher() | |
| chunks = morpher.morph_text("api.md", "# Auth\n\nUse Bearer tokens in the Authorization header.") | |
| index = HybridIndex() | |
| index.add_many(chunks) | |
| toolkit = DocMorphAgentToolkit(index) | |
| schema = toolkit.tool_schema() | |
| assert schema["function"]["name"] == "search_knowledge_base" | |
| output = toolkit.execute_tool_call('{"query": "Bearer tokens"}') | |
| assert "api.md (Auth)" in output | |
| assert "Authorization header" in output | |
| messages = [ | |
| { | |
| "role": "assistant", | |
| "tool_calls": [ | |
| {"id": "1", "type": "function", "function": {"name": "search_knowledge_base", "arguments": '{"query":"a"}'}}, | |
| {"id": "1", "type": "function", "function": {"name": "search_knowledge_base", "arguments": '{"query":"b"}'}}, | |
| ], | |
| }, | |
| {"role": "tool", "tool_call_id": "1", "name": "search_knowledge_base", "content": "res_a"}, | |
| {"role": "tool", "tool_call_id": "1", "name": "search_knowledge_base", "content": "res_b"}, | |
| ] | |
| norm = normalize_parallel_tool_calls(messages) | |
| ids = [tc["id"] for tc in norm[0]["tool_calls"]] | |
| assert len(set(ids)) == 2 | |
| assert norm[1]["tool_call_id"] == ids[0] | |
| assert norm[2]["tool_call_id"] == ids[1] | |
| def test_cli_ingest_search_and_inspect(tmp_path: Path, capsys): | |
| docs_dir = tmp_path / "docs" | |
| docs_dir.mkdir() | |
| (docs_dir / "readme.md").write_text( | |
| "# Storage Engine\n\n| Key | Value |\n| --- | --- |\n| wal_sync | true |\n", | |
| encoding="utf-8", | |
| ) | |
| (docs_dir / "metrics.csv").write_text("node,qps\nnode_1,4500\n", encoding="utf-8") | |
| index_file = tmp_path / "index.jsonl" | |
| assert cli_main(["ingest", str(docs_dir), "-o", str(index_file), "--namespace", "infra"]) == 0 | |
| capsys.readouterr() | |
| assert cli_main(["search", str(index_file), "wal_sync", "--namespace", "infra", "--json"]) == 0 | |
| out = capsys.readouterr().out | |
| results = json.loads(out) | |
| assert len(results) >= 1 | |
| assert results[0]["doc_id"] == "readme.md" | |
| assert "wal_sync" in results[0]["content"] | |
| assert cli_main(["inspect", str(docs_dir / "metrics.csv")]) == 0 | |
| inspect_out = capsys.readouterr().out | |
| assert "| node | qps |" in inspect_out | |