Spaces:
Configuration error
Configuration error
ML Assistant
feat: naive fixed-chunk baseline strategy (eval + Compare panel 0) - the honest before picture
3a0dab2 | #!/usr/bin/env python3 | |
| """Evaluation: Recall@5 / Recall@k comparing dense-only vs hybrid vs | |
| hybrid+rerank on eval/eval_qa.jsonl rows {"question", "doc", "page"}. | |
| Prints the active config flags. --with-loop additionally measures the | |
| query-enhancement loop (HyDE + grading + broaden). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import re | |
| import json | |
| from pathlib import Path | |
| import config | |
| from config import COLLECTION_TEXT | |
| EVAL_SETS = { | |
| "qa": "eval_qa.jsonl", | |
| "hard": "eval_hard.jsonl", | |
| "real": "eval_real.jsonl", | |
| } | |
| DEFAULT_SET = "qa" | |
| SAMPLE_PATH = Path(__file__).parent / "eval" / "eval_qa.sample.jsonl" | |
| def load_rows(set_name: str | None = None): | |
| name = set_name or DEFAULT_SET | |
| fname = EVAL_SETS.get(name) | |
| if fname is None: | |
| raise SystemExit(f"unknown eval set '{name}' (available: {', '.join(EVAL_SETS)})") | |
| path = Path(__file__).parent / "eval" / fname | |
| if not path.exists(): | |
| path = SAMPLE_PATH | |
| rows = [] | |
| with open(path, "r", encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| rows.append(json.loads(line)) | |
| return rows | |
| def print_flags(): | |
| print("active config flags:") | |
| for k in ("ENABLE_VISION_ENRICHMENT", "ENABLE_VISUAL_RETRIEVAL", | |
| "ENABLE_QUERY_ENHANCEMENT", "ENABLE_VISION_ANSWERS", | |
| "ENRICH_QA_ONLY", "ENRICH_MAX_PAGES", "RERANK_MODEL", | |
| "EMBED_MODEL", "CHUNK_MAX_TOKENS", | |
| "ENABLE_NAIVE_BASELINE", "NAIVE_CHUNK_CHARS"): | |
| print(f" {k} = {getattr(config, k)}") | |
| def _norm_doc(s: str) -> str: | |
| """Normalize a document name for matching: lowercase, separators unified, | |
| extension stripped. Payload doc_name (from the filename) and eval rows | |
| (display names) use different conventions.""" | |
| s = str(s).strip().lower() | |
| s = re.sub(r"\.[a-z0-9]+$", "", s) | |
| # alphanumeric-only: filename conventions (kx40-manual.pdf) differ from | |
| # display names in eval sheets (KX-40 Manual.pdf) beyond separators | |
| return re.sub(r"[^a-z0-9]+", "", s) | |
| def _match_rank(points, row, k): | |
| """1-based rank of first doc+page-range match within top-k, else 0. | |
| Points may be ScoredPoint objects (.payload) or plain payload dicts.""" | |
| def _payload(p): | |
| return p.payload if hasattr(p, "payload") else p | |
| doc = _norm_doc(row["doc"]) | |
| for i, p in enumerate(points[:k]): | |
| pl = _payload(p) | |
| p0 = pl.get("page") | |
| p1 = pl.get("page_end", p0) | |
| if (_norm_doc(pl.get("doc_name") or "") == doc | |
| and p0 is not None and int(p0) <= int(row["page"]) <= int(p1)): | |
| return i + 1 | |
| return 0 | |
| def hit(points, row, k) -> bool: | |
| return _match_rank(points, row, k) > 0 | |
| def dense_only(retr, row, k): | |
| vec = retr._embed([row["question"]])[0] | |
| return retr.client.query_points( | |
| COLLECTION_TEXT, query=vec, using="dense", limit=k, with_payload=True).points | |
| def hybrid(retr, row, k): | |
| return retr._hybrid_query(row["question"], row["question"], top_k=k) | |
| def hybrid_rerank(retr, row, k): | |
| pts = retr._hybrid_query(row["question"], row["question"], top_k=config.RERANK_POOL) | |
| return retr._rerank(row["question"], pts, top_n=k) | |
| def _strategy_naive(retr, row, k): | |
| """Naive fixed-chunk baseline (only when ENABLE_NAIVE_BASELINE). | |
| Returns retrieve.py-shaped chunk dicts; _match_rank/hit() consume them | |
| unchanged (doc_name + page/page_end). Chunks are dicts, not ScoredPoints | |
| -- _payload() passes dicts straight through.""" | |
| import naive_baseline | |
| return naive_baseline.naive_search(row["question"], k) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--set", dest="eval_set", choices=sorted(EVAL_SETS), | |
| default=DEFAULT_SET, | |
| help="which eval file to run (default: qa)") | |
| ap.add_argument("--with-loop", action="store_true", | |
| help="measure the query-enhancement loop (HyDE + grading + broaden)") | |
| ap.add_argument("-k", type=int, default=10) | |
| args = ap.parse_args() | |
| print_flags() | |
| rows = load_rows(args.eval_set) | |
| print(f"\n[{args.eval_set}] {len(rows)} eval rows loaded") | |
| import retrieve | |
| retr = retrieve.Retriever() | |
| k = args.k | |
| if config.ENABLE_NAIVE_BASELINE: | |
| # naive baseline needs the raw corpus: same discovery as ingest.py | |
| import naive_baseline | |
| pdf_paths = sorted(str(p) for p in config.DATA_DIR.rglob("*.pdf")) | |
| naive_ok = True | |
| try: | |
| naive_baseline.build_naive_index(pdf_paths) | |
| except Exception as e: | |
| naive_ok = False | |
| print(f" [naive] index build failed, strategy disabled: {e}") | |
| strategies = {"dense-only": dense_only, "hybrid": hybrid, "hybrid+rerank": hybrid_rerank} | |
| if config.ENABLE_NAIVE_BASELINE and locals().get("naive_ok", True): | |
| strategies["naive"] = _strategy_naive # table order: dense, hybrid, rerank, naive | |
| cutoffs = [5] + ([k] if k != 5 else []) # guard: k==5 must not collapse keys | |
| results = {name: {c: 0 for c in cutoffs} for name in strategies} | |
| ranks = {name: [] for name in strategies} # first-hit rank within top-5 | |
| misses = {name: [] for name in strategies} | |
| for row in rows: | |
| for name, fn in strategies.items(): | |
| try: | |
| pts = fn(retr, row, k) | |
| r5 = _match_rank(pts, row, 5) | |
| if r5: | |
| results[name][5] += 1 | |
| ranks[name].append(r5) | |
| else: | |
| misses[name].append(row) | |
| if k != 5 and hit(pts, row, k): | |
| results[name][k] += 1 | |
| except Exception as e: | |
| print(f" [{name}] error on '{row['question'][:40]}': {e}") | |
| misses[name].append(row) | |
| kcol = f"{'Recall@' + str(k):>12}" if k != 5 else f"{'':>12}" | |
| print(f"\n{'strategy':<18}{'Recall@5':>10}{kcol}{'MRR@5':>10}") | |
| n = len(rows) | |
| for name, r in results.items(): | |
| r5 = r[5] / n | |
| rk = (r[k] / n) if k != 5 else None | |
| mrr = (sum(1.0 / r for r in ranks[name]) / n) if ranks[name] else 0.0 | |
| line = f"{name:<18}{r5:>10.2%}" | |
| line += f"{rk:>12.2%}" if rk is not None else f"{'':>12}" | |
| line += f"{mrr:>10.3f}" | |
| print(line) | |
| # per-row MISS breakdown | |
| for name, rows_missed in misses.items(): | |
| if rows_missed: | |
| print(f"\nMISS rows [{name}] ({len(rows_missed)}):") | |
| for row in rows_missed: | |
| print(f" - ({row['doc']}, p{row['page']}) {row['question'][:70]}") | |
| if args.with_loop and config.ENABLE_QUERY_ENHANCEMENT: | |
| print("\nquery-enhancement loop (--with-loop):") | |
| for row in rows: | |
| res = retr.retrieve(row["question"], top_n=k) | |
| pages = [(_norm_doc(c.get("doc_name") or ""), | |
| c.get("page"), c.get("page_end", c.get("page"))) | |
| for c in res["chunks"]] | |
| ok = any(d == _norm_doc(row["doc"]) | |
| and a is not None and b is not None | |
| and int(a) <= int(row["page"]) <= int(b) | |
| for d, a, b in pages) | |
| print(f" {'HIT ' if ok else 'MISS'} broadened={res['broadened']} " | |
| f"q='{row['question'][:50]}' -> {pages[:3]}") | |
| retr.close() | |
| if __name__ == "__main__": | |
| main() | |