Spaces:
Runtime error
Runtime error
Download explore_data_files.py from Harshilforworks/rag-backend: direct link, hf CLI and curl.
- Browser
- Download file 9.36 kB
-
https://huggingface.co/spaces/Harshilforworks/rag-backend/resolve/main/explore_data_files.py
- Command line
-
hf download hf://spaces/Harshilforworks/rag-backend/explore_data_files.py
-
curl -L -o explore_data_files.py https://huggingface.co/spaces/Harshilforworks/rag-backend/resolve/main/explore_data_files.py
9.36 kB
| """Explore project data files with plain-English guidance. | |
| Usage: | |
| python explore_data_files.py | |
| python explore_data_files.py --root . | |
| python explore_data_files.py --save-parsed | |
| This script helps you understand: | |
| 1. What files exist under data/ and related output folders. | |
| 2. What each important file is used for. | |
| 3. What is inside Chroma's sqlite catalog (tables and row counts). | |
| 4. A quick summary of the persisted graph file. | |
| 5. Optionally, save parser outputs from catalog PDFs for manual review. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sqlite3 | |
| from datetime import datetime | |
| from pathlib import Path | |
| UUID_SEGMENT_DIR_RE = re.compile(r"data/chroma_db/[0-9a-f-]{36}$", re.IGNORECASE) | |
| CHROMA_SEGMENT_FILE_GUIDE = { | |
| "header.bin": "HNSW index header (format and index settings).", | |
| "length.bin": "Byte-length/offset helper data used by the vector index.", | |
| "link_lists.bin": "HNSW neighbor link graph used for ANN traversal.", | |
| "data_level0.bin": "Vector payload storage for the base HNSW layer.", | |
| "index_metadata.pickle": "Python metadata blob for ID mappings and index internals.", | |
| } | |
| def human_size(num_bytes: int) -> str: | |
| units = ["B", "KB", "MB", "GB", "TB"] | |
| size = float(max(0, num_bytes)) | |
| for unit in units: | |
| if size < 1024.0 or unit == units[-1]: | |
| if unit == "B": | |
| return f"{int(size)} {unit}" | |
| return f"{size:.2f} {unit}" | |
| size /= 1024.0 | |
| return f"{num_bytes} B" | |
| def print_header(title: str) -> None: | |
| bar = "=" * len(title) | |
| print(f"\n{title}\n{bar}") | |
| def rel_path(path: Path, root: Path) -> str: | |
| try: | |
| return path.relative_to(root).as_posix() | |
| except ValueError: | |
| return path.as_posix() | |
| def describe_entry(path: Path, root: Path) -> str: | |
| rel = rel_path(path, root) | |
| if path.is_dir(): | |
| if rel == "data": | |
| return "Main persisted runtime data folder." | |
| if rel == "data/catalog_docs": | |
| return "Source PDFs used for ingestion." | |
| if rel == "data/chroma_db": | |
| return "Chroma persistent store root (sqlite + vector index segments)." | |
| if UUID_SEGMENT_DIR_RE.match(rel): | |
| return "One Chroma vector segment folder (HNSW index files)." | |
| if rel == "output": | |
| return "Saved parser output previews (if enabled)." | |
| if rel == "request_logs": | |
| return "Per-request trace snapshots for API calls." | |
| if rel == "logs": | |
| return "Application log files." | |
| return "Directory" | |
| name = path.name | |
| if rel == "data/chroma_db/chroma.sqlite3": | |
| return "Chroma sqlite catalog (collections, segments, metadata, document refs)." | |
| if rel == "data/graph_store.json": | |
| return "Persisted prerequisite graph (nodes/edges)." | |
| if rel.startswith("data/catalog_docs/") and path.suffix.lower() == ".pdf": | |
| return "Input PDF used by parser/chunker during ingestion." | |
| if rel.startswith("output/") and path.suffix.lower() == ".txt": | |
| return "Saved parsed text output from hybrid parser." | |
| if rel.startswith("request_logs/") and path.suffix.lower() == ".json": | |
| return "Captured request/response context for debugging." | |
| parent_rel = rel_path(path.parent, root) | |
| if UUID_SEGMENT_DIR_RE.match(parent_rel): | |
| return CHROMA_SEGMENT_FILE_GUIDE.get(name, "Chroma segment artifact file.") | |
| return "File" | |
| def list_relevant_entries(root: Path) -> list[Path]: | |
| paths: list[Path] = [] | |
| for folder in ("data", "output", "logs", "request_logs"): | |
| base = root / folder | |
| if not base.exists(): | |
| continue | |
| for path in sorted(base.rglob("*")): | |
| paths.append(path) | |
| return paths | |
| def print_filesystem_guide(root: Path) -> None: | |
| print_header("Filesystem Guide") | |
| entries = list_relevant_entries(root) | |
| if not entries: | |
| print("No data/output/log files were found yet.") | |
| return | |
| for path in entries: | |
| rel = rel_path(path, root) | |
| role = describe_entry(path, root) | |
| if path.is_dir(): | |
| print(f"[DIR] {rel}") | |
| print(f" -> {role}") | |
| else: | |
| stat = path.stat() | |
| mtime = datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds") | |
| print(f"[FILE] {rel} | {human_size(stat.st_size)} | modified {mtime}") | |
| print(f" -> {role}") | |
| def sql_columns(cur: sqlite3.Cursor, table: str) -> list[str]: | |
| quoted = table.replace("\"", "\"\"") | |
| rows = cur.execute(f'PRAGMA table_info("{quoted}")').fetchall() | |
| return [row[1] for row in rows] | |
| def safe_count(cur: sqlite3.Cursor, table: str) -> int | None: | |
| quoted = table.replace("\"", "\"\"") | |
| try: | |
| return int(cur.execute(f'SELECT COUNT(*) FROM "{quoted}"').fetchone()[0]) | |
| except Exception: | |
| return None | |
| def inspect_chroma_sqlite(root: Path, sample_rows: int) -> None: | |
| print_header("Chroma SQLite Inspection") | |
| db_path = root / "data" / "chroma_db" / "chroma.sqlite3" | |
| if not db_path.exists(): | |
| print("No chroma sqlite file found at data/chroma_db/chroma.sqlite3") | |
| return | |
| print(f"Database: {rel_path(db_path, root)}") | |
| conn = sqlite3.connect(str(db_path)) | |
| cur = conn.cursor() | |
| try: | |
| tables = [ | |
| row[0] | |
| for row in cur.execute( | |
| "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" | |
| ).fetchall() | |
| ] | |
| print(f"Tables ({len(tables)}): {', '.join(tables)}") | |
| print("\nTable row counts:") | |
| for table in tables: | |
| count = safe_count(cur, table) | |
| if count is None: | |
| print(f"- {table}: unable to count") | |
| else: | |
| print(f"- {table}: {count}") | |
| if "collections" in tables: | |
| cols = sql_columns(cur, "collections") | |
| wanted = [c for c in ("id", "name", "dimension", "database_id") if c in cols] | |
| if wanted: | |
| rows = cur.execute( | |
| f"SELECT {', '.join(wanted)} FROM collections LIMIT ?", | |
| (sample_rows,), | |
| ).fetchall() | |
| print("\nCollections sample:") | |
| for row in rows: | |
| print(f"- {dict(zip(wanted, row))}") | |
| if "segments" in tables: | |
| cols = sql_columns(cur, "segments") | |
| wanted = [c for c in ("id", "type", "scope", "collection") if c in cols] | |
| if wanted: | |
| rows = cur.execute( | |
| f"SELECT {', '.join(wanted)} FROM segments LIMIT ?", | |
| (sample_rows,), | |
| ).fetchall() | |
| print("\nSegments sample:") | |
| for row in rows: | |
| print(f"- {dict(zip(wanted, row))}") | |
| finally: | |
| conn.close() | |
| def inspect_graph_store(root: Path) -> None: | |
| print_header("Graph Store Inspection") | |
| graph_path = root / "data" / "graph_store.json" | |
| if not graph_path.exists(): | |
| print("No graph store found at data/graph_store.json") | |
| return | |
| try: | |
| with graph_path.open("r", encoding="utf-8") as f: | |
| payload = json.load(f) | |
| nodes = payload.get("nodes", []) if isinstance(payload, dict) else [] | |
| edges = [] | |
| if isinstance(payload, dict): | |
| edges = payload.get("links", payload.get("edges", [])) | |
| print(f"Path: {rel_path(graph_path, root)}") | |
| print(f"Nodes: {len(nodes)}") | |
| print(f"Edges: {len(edges)}") | |
| if nodes: | |
| sample = nodes[:5] | |
| print("Sample node entries (up to 5):") | |
| for node in sample: | |
| if isinstance(node, dict): | |
| label = node.get("id", "<no id>") | |
| else: | |
| label = str(node) | |
| print(f"- {label}") | |
| except Exception as exc: | |
| print(f"Failed to read graph store: {exc}") | |
| def save_parser_outputs(root: Path) -> None: | |
| print_header("Saving Parser Outputs") | |
| try: | |
| from src.processing.parser import parse_document_hybrid | |
| except Exception as exc: | |
| print("Could not import parser from src.processing.parser") | |
| print(f"Import error: {exc}") | |
| print("Run this script from the project root where src/ is present.") | |
| return | |
| catalog_dir = root / "data" / "catalog_docs" | |
| if not catalog_dir.exists(): | |
| print("Catalog directory not found: data/catalog_docs") | |
| return | |
| pdfs = sorted(catalog_dir.glob("*.pdf")) | |
| if not pdfs: | |
| print("No PDF files found in data/catalog_docs") | |
| return | |
| print(f"Found {len(pdfs)} PDF(s). Parsing and saving outputs...") | |
| for pdf in pdfs: | |
| print(f"- Parsing {pdf.name}") | |
| parse_document_hybrid(str(pdf), save_parsed_text=True) | |
| print("Saved parser outputs to output/*.txt") | |
| def build_arg_parser() -> argparse.ArgumentParser: | |
| parser = argparse.ArgumentParser( | |
| description="Explore project data files and explain what each file does." | |
| ) | |
| parser.add_argument( | |
| "--root", | |
| default=".", | |
| help="Project root path (defaults to current directory).", | |
| ) | |
| parser.add_argument( | |
| "--sample-rows", | |
| type=int, | |
| default=5, | |
| help="How many sample rows to print from sqlite tables.", | |
| ) | |
| parser.add_argument( | |
| "--save-parsed", | |
| action="store_true", | |
| help="Parse PDFs from data/catalog_docs and save parser outputs to output/*.txt.", | |
| ) | |
| return parser | |
| def main() -> None: | |
| args = build_arg_parser().parse_args() | |
| root = Path(args.root).resolve() | |
| print_header("Data Explorer") | |
| print(f"Project root: {root}") | |
| print("This report explains file purpose and basic contents.") | |
| print_filesystem_guide(root) | |
| inspect_chroma_sqlite(root, max(1, args.sample_rows)) | |
| inspect_graph_store(root) | |
| if args.save_parsed: | |
| save_parser_outputs(root) | |
| print_header("Quick Tips") | |
| print("- Run with --save-parsed to generate human-readable parser outputs in output/*.txt") | |
| print("- Compare output/*_parsed.txt with source PDFs to validate extraction quality.") | |
| print("- If Chroma row counts grow but retrieval quality drops, inspect chunk metadata fields.") | |
| if __name__ == "__main__": | |
| main() | |