Spaces:
Running
Running
Shoufa Zhang
Deploy DocMorph & SeekMorph interactive Studio and 5-layer production benchmark
66227f0 Download docmorph/cli.py from zhangshoufa/DocMorph: direct link, hf CLI and curl.
- Browser
- Download file 4.25 kB
-
https://huggingface.co/spaces/zhangshoufa/DocMorph/resolve/main/docmorph/cli.py
- Command line
-
hf download hf://spaces/zhangshoufa/DocMorph/docmorph/cli.py
-
curl -L -o cli.py https://huggingface.co/spaces/zhangshoufa/DocMorph/resolve/main/docmorph/cli.py
4.25 kB
| """Command-line interface for DocMorph: ingest, search, and inspect documents.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from docmorph.index import HybridIndex | |
| from docmorph.parser import DocumentMorpher | |
| def build_parser() -> argparse.ArgumentParser: | |
| parser = argparse.ArgumentParser( | |
| prog="docmorph", | |
| description="Structure-aware document chunking and hybrid BM25/vector retrieval for AI Agents.", | |
| ) | |
| sub = parser.add_subparsers(dest="command", required=True) | |
| p_ingest = sub.add_parser("ingest", help="Parse and chunk a file or directory into a JSONL hybrid index.") | |
| p_ingest.add_argument("source", help="Path to a document file or directory.") | |
| p_ingest.add_argument("-o", "--output", required=True, help="Output JSONL index path.") | |
| p_ingest.add_argument("--max-chars", type=int, default=1000, help="Maximum characters per chunk (default: 1000).") | |
| p_ingest.add_argument("--namespace", default="default", help="Namespace tag for indexed chunks (default: default).") | |
| p_search = sub.add_parser("search", help="Search a JSONL index with BM25 + RRF ranking.") | |
| p_search.add_argument("index_path", help="Path to the JSONL index file.") | |
| p_search.add_argument("query", help="Search query string.") | |
| p_search.add_argument("-k", "--top-k", type=int, default=5, help="Number of results to return (default: 5).") | |
| p_search.add_argument("--namespace", default=None, help="Optional namespace filter.") | |
| p_search.add_argument( | |
| "-w", "--expand-window", type=int, default=0, help="Neighboring chunks in the same section to expand." | |
| ) | |
| p_search.add_argument("--json", action="store_true", help="Emit results as JSON.") | |
| p_inspect = sub.add_parser("inspect", help="Preview how DocMorph parses and chunks a file.") | |
| p_inspect.add_argument("file_path", help="Path to a document file (.md, .html, .csv, .tsv, .json, .txt).") | |
| p_inspect.add_argument("--max-chars", type=int, default=1000, help="Maximum characters per chunk (default: 1000).") | |
| return parser | |
| def main(argv: list[str] | None = None) -> int: | |
| args = build_parser().parse_args(argv) | |
| if args.command == "ingest": | |
| src = Path(args.source) | |
| if not src.exists(): | |
| print(f"Error: source path does not exist: {src}", file=sys.stderr) | |
| return 1 | |
| morpher = DocumentMorpher(max_chars=args.max_chars) | |
| chunks = morpher.morph_directory(src, namespace=args.namespace) | |
| index = HybridIndex() | |
| index.add_many(chunks) | |
| index.save(args.output) | |
| print(f"Indexed {len(chunks)} structure-aware chunks from '{src}' -> '{args.output}'") | |
| return 0 | |
| if args.command == "search": | |
| index = HybridIndex.load(args.index_path) | |
| hits = index.search( | |
| args.query, | |
| top_k=args.top_k, | |
| namespace=args.namespace, | |
| expand_window=args.expand_window, | |
| ) | |
| if args.json: | |
| payload = [ | |
| { | |
| "rank": i + 1, | |
| "chunk_id": h.chunk.chunk_id, | |
| "doc_id": h.chunk.doc_id, | |
| "section": list(h.chunk.section_path), | |
| "kind": h.chunk.kind, | |
| "score": h.score, | |
| "bm25_score": h.bm25_score, | |
| "content": h.expanded_content, | |
| } | |
| for i, h in enumerate(hits) | |
| ] | |
| print(json.dumps(payload, ensure_ascii=False, indent=2)) | |
| else: | |
| if not hits: | |
| print("No results found.") | |
| return 0 | |
| for i, h in enumerate(hits): | |
| print(f"{h.format_citation(rank=i + 1)} (bm25={h.bm25_score})") | |
| if i + 1 < len(hits): | |
| print("\n" + "-" * 60 + "\n") | |
| return 0 | |
| if args.command == "inspect": | |
| morpher = DocumentMorpher(max_chars=args.max_chars) | |
| chunks = morpher.morph_file(args.file_path) | |
| for c in chunks: | |
| print(f"[{c.chunk_index}] ({c.kind}) {c.breadcrumb_str} [{len(c.content)} chars]") | |
| print(c.content) | |
| print("-" * 50) | |
| return 0 | |
| return 1 | |