ResearchOS / scripts /review_failures.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
3.51 kB
"""Read the failure log. Run this weekly once people are using the app.
python scripts/review_failures.py --config configs/ml.yaml
Five things worth looking at, in the order they usually matter:
errors exceptions. Anything here is a bug, not a limitation.
downvoted someone said the answer was wrong. Read the query.
out_of_corpus the retriever declined. If a topic keeps appearing, that is
the corpus telling you what to ingest next.
never_clicked confident answers nobody opened a citation for. Weak signal
individually; a pattern means results look irrelevant.
slowest where the latency actually goes.
"""
from __future__ import annotations
import argparse
import collections
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.api.db import FeedbackLog
from src.config import load_config
def section(title: str, rows: list[dict], columns: list[str], limit: int) -> None:
print(f"\n{'=' * 78}\n{title} ({len(rows)})\n{'=' * 78}")
if not rows:
print(" nothing recorded")
return
for row in rows[:limit]:
parts = [str(row.get(c, ""))[:96] for c in columns if row.get(c) not in (None, "")]
print(" " + " | ".join(parts))
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", default="configs/ml.yaml")
parser.add_argument("--limit", type=int, default=15)
args = parser.parse_args()
cfg = load_config(args.config)
log = FeedbackLog(cfg["paths"]["feedback_db"])
stats = log.stats()
print("\nUSAGE")
for key, value in stats.items():
print(f" {key:16s} {value}")
total = stats["queries"]
if total:
low = stats["low_confidence"] / total * 100
clicked = stats["clicks"] / total
print(f"\n {low:.0f}% of queries were out-of-corpus")
print(f" {clicked:.2f} citation clicks per query")
if stats["votes_up"] + stats["votes_down"]:
up = stats["votes_up"] / (stats["votes_up"] + stats["votes_down"]) * 100
print(f" {up:.0f}% of votes were positive")
data = log.recent_failures(limit=200)
section("ERRORS - these are bugs", data["errors"],
["ts", "endpoint", "error_type", "message"], args.limit)
section("DOWNVOTED - the answer was wrong", data["downvoted"],
["ts", "mode", "query", "comment"], args.limit)
section("OUT OF CORPUS - what people want that you have not ingested",
data["out_of_corpus"], ["ts", "mode", "query", "mean_top_score"], args.limit)
section("CONFIDENT BUT NO CLICKS - results may look irrelevant",
data["never_clicked"], ["ts", "mode", "query"], args.limit)
section("SLOWEST", data["slowest"], ["latency_ms", "mode", "query"], args.limit)
if data["out_of_corpus"]:
words = collections.Counter()
for row in data["out_of_corpus"]:
for word in str(row.get("query", "")).lower().split():
if len(word) > 4:
words[word] += 1
common = words.most_common(10)
if common:
print("\nMost common terms in out-of-corpus queries - candidate ingest topics:")
print(" " + ", ".join(f"{w} ({n})" for w, n in common))
print()
if __name__ == "__main__":
main()