Spaces:
Running on Zero
Running on Zero
Download scripts/review_failures.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 3.51 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/review_failures.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/review_failures.py
-
curl -L -o review_failures.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/review_failures.py
3.51 kB
| """Read the failure log. Run this weekly once people are using the app. | |
| python scripts/review_failures.py --config configs/ml.yaml | |
| Five things worth looking at, in the order they usually matter: | |
| errors exceptions. Anything here is a bug, not a limitation. | |
| downvoted someone said the answer was wrong. Read the query. | |
| out_of_corpus the retriever declined. If a topic keeps appearing, that is | |
| the corpus telling you what to ingest next. | |
| never_clicked confident answers nobody opened a citation for. Weak signal | |
| individually; a pattern means results look irrelevant. | |
| slowest where the latency actually goes. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| from src.api.db import FeedbackLog | |
| from src.config import load_config | |
| def section(title: str, rows: list[dict], columns: list[str], limit: int) -> None: | |
| print(f"\n{'=' * 78}\n{title} ({len(rows)})\n{'=' * 78}") | |
| if not rows: | |
| print(" nothing recorded") | |
| return | |
| for row in rows[:limit]: | |
| parts = [str(row.get(c, ""))[:96] for c in columns if row.get(c) not in (None, "")] | |
| print(" " + " | ".join(parts)) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", default="configs/ml.yaml") | |
| parser.add_argument("--limit", type=int, default=15) | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| log = FeedbackLog(cfg["paths"]["feedback_db"]) | |
| stats = log.stats() | |
| print("\nUSAGE") | |
| for key, value in stats.items(): | |
| print(f" {key:16s} {value}") | |
| total = stats["queries"] | |
| if total: | |
| low = stats["low_confidence"] / total * 100 | |
| clicked = stats["clicks"] / total | |
| print(f"\n {low:.0f}% of queries were out-of-corpus") | |
| print(f" {clicked:.2f} citation clicks per query") | |
| if stats["votes_up"] + stats["votes_down"]: | |
| up = stats["votes_up"] / (stats["votes_up"] + stats["votes_down"]) * 100 | |
| print(f" {up:.0f}% of votes were positive") | |
| data = log.recent_failures(limit=200) | |
| section("ERRORS - these are bugs", data["errors"], | |
| ["ts", "endpoint", "error_type", "message"], args.limit) | |
| section("DOWNVOTED - the answer was wrong", data["downvoted"], | |
| ["ts", "mode", "query", "comment"], args.limit) | |
| section("OUT OF CORPUS - what people want that you have not ingested", | |
| data["out_of_corpus"], ["ts", "mode", "query", "mean_top_score"], args.limit) | |
| section("CONFIDENT BUT NO CLICKS - results may look irrelevant", | |
| data["never_clicked"], ["ts", "mode", "query"], args.limit) | |
| section("SLOWEST", data["slowest"], ["latency_ms", "mode", "query"], args.limit) | |
| if data["out_of_corpus"]: | |
| words = collections.Counter() | |
| for row in data["out_of_corpus"]: | |
| for word in str(row.get("query", "")).lower().split(): | |
| if len(word) > 4: | |
| words[word] += 1 | |
| common = words.most_common(10) | |
| if common: | |
| print("\nMost common terms in out-of-corpus queries - candidate ingest topics:") | |
| print(" " + ", ".join(f"{w} ({n})" for w, n in common)) | |
| print() | |
| if __name__ == "__main__": | |
| main() | |