File size: 6,211 Bytes
66227f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
"""End-to-end tests for DocMorph parser, table header preservation, hybrid index, agent toolkit, and CLI."""

from __future__ import annotations

import json
from pathlib import Path

from docmorph import (
    DocMorphAgentToolkit,
    DocumentMorpher,
    HybridIndex,
    csv_to_markdown_table,
    html_to_markdown,
    normalize_parallel_tool_calls,
)
from docmorph.cli import main as cli_main


def test_html_to_markdown_converts_headings_tables_and_code():
    html = """
    <html>
      <head><script>console.log("ignore")</script></head>
      <body>
        <h1>Deployment Guide</h1>
        <p>Run the server with speculative decoding enabled.</p>
        <table>
          <tr><th>Hardware</th><th>Acceptance</th></tr>
          <tr><td>MI308X</td><td>3.12</td></tr>
        </table>
        <pre><code>vllm serve deepseek-v4</code></pre>
      </body>
    </html>
    """
    md = html_to_markdown(html)
    assert "# Deployment Guide" in md
    assert "| Hardware | Acceptance |" in md
    assert "| MI308X | 3.12 |" in md
    assert "vllm serve deepseek-v4" in md
    assert "console.log" not in md


def test_large_table_splitting_repeats_header_in_every_chunk():
    rows = ["metric,q1,q2"] + [f"row_{i},100_{i},200_{i}" for i in range(25)]
    csv_text = "\n".join(rows)
    morpher = DocumentMorpher(max_chars=180)
    chunks = morpher.morph_text("financials.csv", csv_text, fmt="csv")

    assert len(chunks) >= 3
    for chunk in chunks:
        assert chunk.kind == "table"
        lines = chunk.content.splitlines()
        assert lines[0] == "| metric | q1 | q2 |"
        assert lines[1] == "| --- | --- | --- |"


def test_hybrid_index_multilingual_bm25_and_dense_rrf_with_persistence(tmp_path: Path):
    morpher = DocumentMorpher(max_chars=400)
    md = (
        "# 架构设计\n\n"
        "## 投机采样加速\n\n"
        "DeepSeek-V4 使用 DSpark 投机采样,在 8 卡 MI308X 上实现 3.1 倍吞吐提升。\n\n"
        "## 混合检索模块\n\n"
        "结合 BM25 词法匹配与稠密向量余弦相似度,通过 RRF 融合排序。\n"
    )
    chunks = morpher.morph_text("design.md", md, namespace="prod")
    assert len(chunks) == 2

    index = HybridIndex()
    index.add(chunks[0], embedding=[0.9, 0.1])
    index.add(chunks[1], embedding=[0.1, 0.9])

    index_path = tmp_path / "kb.jsonl"
    index.save(index_path)

    # Append a corrupted partial line to verify fault-tolerant loading
    with index_path.open("a", encoding="utf-8") as f:
        f.write("{corrupted_json_line\n")

    reloaded = HybridIndex.load(index_path)
    assert len(reloaded) == 2

    hits = reloaded.search("投机采样 MI308X", top_k=1, namespace="prod", query_embedding=[0.95, 0.05])
    assert len(hits) == 1
    assert hits[0].chunk.section_path == ("架构设计", "投机采样加速")
    assert "3.1 倍" in hits[0].expanded_content

    assert reloaded.remove_document("design.md", namespace="prod") == 2
    assert len(reloaded) == 0
    assert reloaded.search("投机采样 MI308X", top_k=1, namespace="prod") == []


def test_context_window_expansion_within_same_section():
    morpher = DocumentMorpher(max_chars=90)
    md = (
        "# Section A\n\n"
        "Paragraph one introduces the system configuration parameters and baseline setup.\n\n"
        "Paragraph two explains the exact token acceptance threshold and fallback behavior.\n\n"
        "Paragraph three summarizes the final benchmark numbers on production traffic.\n"
    )
    chunks = morpher.morph_text("guide.md", md)
    assert len(chunks) == 3

    index = HybridIndex()
    index.add_many(chunks)

    hits = index.search("acceptance threshold", top_k=1, expand_window=1)
    assert len(hits) == 1
    assert "Paragraph one" in hits[0].expanded_content
    assert "Paragraph two" in hits[0].expanded_content
    assert "Paragraph three" in hits[0].expanded_content


def test_agent_toolkit_and_parallel_tool_call_normalization():
    morpher = DocumentMorpher()
    chunks = morpher.morph_text("api.md", "# Auth\n\nUse Bearer tokens in the Authorization header.")
    index = HybridIndex()
    index.add_many(chunks)

    toolkit = DocMorphAgentToolkit(index)
    schema = toolkit.tool_schema()
    assert schema["function"]["name"] == "search_knowledge_base"

    output = toolkit.execute_tool_call('{"query": "Bearer tokens"}')
    assert "api.md (Auth)" in output
    assert "Authorization header" in output

    messages = [
        {
            "role": "assistant",
            "tool_calls": [
                {"id": "1", "type": "function", "function": {"name": "search_knowledge_base", "arguments": '{"query":"a"}'}},
                {"id": "1", "type": "function", "function": {"name": "search_knowledge_base", "arguments": '{"query":"b"}'}},
            ],
        },
        {"role": "tool", "tool_call_id": "1", "name": "search_knowledge_base", "content": "res_a"},
        {"role": "tool", "tool_call_id": "1", "name": "search_knowledge_base", "content": "res_b"},
    ]
    norm = normalize_parallel_tool_calls(messages)
    ids = [tc["id"] for tc in norm[0]["tool_calls"]]
    assert len(set(ids)) == 2
    assert norm[1]["tool_call_id"] == ids[0]
    assert norm[2]["tool_call_id"] == ids[1]


def test_cli_ingest_search_and_inspect(tmp_path: Path, capsys):
    docs_dir = tmp_path / "docs"
    docs_dir.mkdir()
    (docs_dir / "readme.md").write_text(
        "# Storage Engine\n\n| Key | Value |\n| --- | --- |\n| wal_sync | true |\n",
        encoding="utf-8",
    )
    (docs_dir / "metrics.csv").write_text("node,qps\nnode_1,4500\n", encoding="utf-8")

    index_file = tmp_path / "index.jsonl"
    assert cli_main(["ingest", str(docs_dir), "-o", str(index_file), "--namespace", "infra"]) == 0
    capsys.readouterr()

    assert cli_main(["search", str(index_file), "wal_sync", "--namespace", "infra", "--json"]) == 0
    out = capsys.readouterr().out
    results = json.loads(out)
    assert len(results) >= 1
    assert results[0]["doc_id"] == "readme.md"
    assert "wal_sync" in results[0]["content"]

    assert cli_main(["inspect", str(docs_dir / "metrics.csv")]) == 0
    inspect_out = capsys.readouterr().out
    assert "| node | qps |" in inspect_out