"""End-to-end tests for DocMorph parser, table header preservation, hybrid index, agent toolkit, and CLI.""" from __future__ import annotations import json from pathlib import Path from docmorph import ( DocMorphAgentToolkit, DocumentMorpher, HybridIndex, csv_to_markdown_table, html_to_markdown, normalize_parallel_tool_calls, ) from docmorph.cli import main as cli_main def test_html_to_markdown_converts_headings_tables_and_code(): html = """

Deployment Guide

Run the server with speculative decoding enabled.

HardwareAcceptance
MI308X3.12
vllm serve deepseek-v4
""" md = html_to_markdown(html) assert "# Deployment Guide" in md assert "| Hardware | Acceptance |" in md assert "| MI308X | 3.12 |" in md assert "vllm serve deepseek-v4" in md assert "console.log" not in md def test_large_table_splitting_repeats_header_in_every_chunk(): rows = ["metric,q1,q2"] + [f"row_{i},100_{i},200_{i}" for i in range(25)] csv_text = "\n".join(rows) morpher = DocumentMorpher(max_chars=180) chunks = morpher.morph_text("financials.csv", csv_text, fmt="csv") assert len(chunks) >= 3 for chunk in chunks: assert chunk.kind == "table" lines = chunk.content.splitlines() assert lines[0] == "| metric | q1 | q2 |" assert lines[1] == "| --- | --- | --- |" def test_hybrid_index_multilingual_bm25_and_dense_rrf_with_persistence(tmp_path: Path): morpher = DocumentMorpher(max_chars=400) md = ( "# 架构设计\n\n" "## 投机采样加速\n\n" "DeepSeek-V4 使用 DSpark 投机采样,在 8 卡 MI308X 上实现 3.1 倍吞吐提升。\n\n" "## 混合检索模块\n\n" "结合 BM25 词法匹配与稠密向量余弦相似度,通过 RRF 融合排序。\n" ) chunks = morpher.morph_text("design.md", md, namespace="prod") assert len(chunks) == 2 index = HybridIndex() index.add(chunks[0], embedding=[0.9, 0.1]) index.add(chunks[1], embedding=[0.1, 0.9]) index_path = tmp_path / "kb.jsonl" index.save(index_path) # Append a corrupted partial line to verify fault-tolerant loading with index_path.open("a", encoding="utf-8") as f: f.write("{corrupted_json_line\n") reloaded = HybridIndex.load(index_path) assert len(reloaded) == 2 hits = reloaded.search("投机采样 MI308X", top_k=1, namespace="prod", query_embedding=[0.95, 0.05]) assert len(hits) == 1 assert hits[0].chunk.section_path == ("架构设计", "投机采样加速") assert "3.1 倍" in hits[0].expanded_content assert reloaded.remove_document("design.md", namespace="prod") == 2 assert len(reloaded) == 0 assert reloaded.search("投机采样 MI308X", top_k=1, namespace="prod") == [] def test_context_window_expansion_within_same_section(): morpher = DocumentMorpher(max_chars=90) md = ( "# Section A\n\n" "Paragraph one introduces the system configuration parameters and baseline setup.\n\n" "Paragraph two explains the exact token acceptance threshold and fallback behavior.\n\n" "Paragraph three summarizes the final benchmark numbers on production traffic.\n" ) chunks = morpher.morph_text("guide.md", md) assert len(chunks) == 3 index = HybridIndex() index.add_many(chunks) hits = index.search("acceptance threshold", top_k=1, expand_window=1) assert len(hits) == 1 assert "Paragraph one" in hits[0].expanded_content assert "Paragraph two" in hits[0].expanded_content assert "Paragraph three" in hits[0].expanded_content def test_agent_toolkit_and_parallel_tool_call_normalization(): morpher = DocumentMorpher() chunks = morpher.morph_text("api.md", "# Auth\n\nUse Bearer tokens in the Authorization header.") index = HybridIndex() index.add_many(chunks) toolkit = DocMorphAgentToolkit(index) schema = toolkit.tool_schema() assert schema["function"]["name"] == "search_knowledge_base" output = toolkit.execute_tool_call('{"query": "Bearer tokens"}') assert "api.md (Auth)" in output assert "Authorization header" in output messages = [ { "role": "assistant", "tool_calls": [ {"id": "1", "type": "function", "function": {"name": "search_knowledge_base", "arguments": '{"query":"a"}'}}, {"id": "1", "type": "function", "function": {"name": "search_knowledge_base", "arguments": '{"query":"b"}'}}, ], }, {"role": "tool", "tool_call_id": "1", "name": "search_knowledge_base", "content": "res_a"}, {"role": "tool", "tool_call_id": "1", "name": "search_knowledge_base", "content": "res_b"}, ] norm = normalize_parallel_tool_calls(messages) ids = [tc["id"] for tc in norm[0]["tool_calls"]] assert len(set(ids)) == 2 assert norm[1]["tool_call_id"] == ids[0] assert norm[2]["tool_call_id"] == ids[1] def test_cli_ingest_search_and_inspect(tmp_path: Path, capsys): docs_dir = tmp_path / "docs" docs_dir.mkdir() (docs_dir / "readme.md").write_text( "# Storage Engine\n\n| Key | Value |\n| --- | --- |\n| wal_sync | true |\n", encoding="utf-8", ) (docs_dir / "metrics.csv").write_text("node,qps\nnode_1,4500\n", encoding="utf-8") index_file = tmp_path / "index.jsonl" assert cli_main(["ingest", str(docs_dir), "-o", str(index_file), "--namespace", "infra"]) == 0 capsys.readouterr() assert cli_main(["search", str(index_file), "wal_sync", "--namespace", "infra", "--json"]) == 0 out = capsys.readouterr().out results = json.loads(out) assert len(results) >= 1 assert results[0]["doc_id"] == "readme.md" assert "wal_sync" in results[0]["content"] assert cli_main(["inspect", str(docs_dir / "metrics.csv")]) == 0 inspect_out = capsys.readouterr().out assert "| node | qps |" in inspect_out