File size: 4,253 Bytes
66227f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
"""Command-line interface for DocMorph: ingest, search, and inspect documents."""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

from docmorph.index import HybridIndex
from docmorph.parser import DocumentMorpher


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        prog="docmorph",
        description="Structure-aware document chunking and hybrid BM25/vector retrieval for AI Agents.",
    )
    sub = parser.add_subparsers(dest="command", required=True)

    p_ingest = sub.add_parser("ingest", help="Parse and chunk a file or directory into a JSONL hybrid index.")
    p_ingest.add_argument("source", help="Path to a document file or directory.")
    p_ingest.add_argument("-o", "--output", required=True, help="Output JSONL index path.")
    p_ingest.add_argument("--max-chars", type=int, default=1000, help="Maximum characters per chunk (default: 1000).")
    p_ingest.add_argument("--namespace", default="default", help="Namespace tag for indexed chunks (default: default).")

    p_search = sub.add_parser("search", help="Search a JSONL index with BM25 + RRF ranking.")
    p_search.add_argument("index_path", help="Path to the JSONL index file.")
    p_search.add_argument("query", help="Search query string.")
    p_search.add_argument("-k", "--top-k", type=int, default=5, help="Number of results to return (default: 5).")
    p_search.add_argument("--namespace", default=None, help="Optional namespace filter.")
    p_search.add_argument(
        "-w", "--expand-window", type=int, default=0, help="Neighboring chunks in the same section to expand."
    )
    p_search.add_argument("--json", action="store_true", help="Emit results as JSON.")

    p_inspect = sub.add_parser("inspect", help="Preview how DocMorph parses and chunks a file.")
    p_inspect.add_argument("file_path", help="Path to a document file (.md, .html, .csv, .tsv, .json, .txt).")
    p_inspect.add_argument("--max-chars", type=int, default=1000, help="Maximum characters per chunk (default: 1000).")

    return parser


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)

    if args.command == "ingest":
        src = Path(args.source)
        if not src.exists():
            print(f"Error: source path does not exist: {src}", file=sys.stderr)
            return 1
        morpher = DocumentMorpher(max_chars=args.max_chars)
        chunks = morpher.morph_directory(src, namespace=args.namespace)
        index = HybridIndex()
        index.add_many(chunks)
        index.save(args.output)
        print(f"Indexed {len(chunks)} structure-aware chunks from '{src}' -> '{args.output}'")
        return 0

    if args.command == "search":
        index = HybridIndex.load(args.index_path)
        hits = index.search(
            args.query,
            top_k=args.top_k,
            namespace=args.namespace,
            expand_window=args.expand_window,
        )
        if args.json:
            payload = [
                {
                    "rank": i + 1,
                    "chunk_id": h.chunk.chunk_id,
                    "doc_id": h.chunk.doc_id,
                    "section": list(h.chunk.section_path),
                    "kind": h.chunk.kind,
                    "score": h.score,
                    "bm25_score": h.bm25_score,
                    "content": h.expanded_content,
                }
                for i, h in enumerate(hits)
            ]
            print(json.dumps(payload, ensure_ascii=False, indent=2))
        else:
            if not hits:
                print("No results found.")
                return 0
            for i, h in enumerate(hits):
                print(f"{h.format_citation(rank=i + 1)}  (bm25={h.bm25_score})")
                if i + 1 < len(hits):
                    print("\n" + "-" * 60 + "\n")
        return 0

    if args.command == "inspect":
        morpher = DocumentMorpher(max_chars=args.max_chars)
        chunks = morpher.morph_file(args.file_path)
        for c in chunks:
            print(f"[{c.chunk_index}] ({c.kind}) {c.breadcrumb_str} [{len(c.content)} chars]")
            print(c.content)
            print("-" * 50)
        return 0

    return 1