File size: 3,510 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
"""Read the failure log. Run this weekly once people are using the app.

    python scripts/review_failures.py --config configs/ml.yaml

Five things worth looking at, in the order they usually matter:

  errors          exceptions. Anything here is a bug, not a limitation.
  downvoted       someone said the answer was wrong. Read the query.
  out_of_corpus   the retriever declined. If a topic keeps appearing, that is
                  the corpus telling you what to ingest next.
  never_clicked   confident answers nobody opened a citation for. Weak signal
                  individually; a pattern means results look irrelevant.
  slowest         where the latency actually goes.
"""

from __future__ import annotations

import argparse
import collections
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from src.api.db import FeedbackLog
from src.config import load_config


def section(title: str, rows: list[dict], columns: list[str], limit: int) -> None:
    print(f"\n{'=' * 78}\n{title}  ({len(rows)})\n{'=' * 78}")
    if not rows:
        print("  nothing recorded")
        return
    for row in rows[:limit]:
        parts = [str(row.get(c, ""))[:96] for c in columns if row.get(c) not in (None, "")]
        print("  " + " | ".join(parts))


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--config", default="configs/ml.yaml")
    parser.add_argument("--limit", type=int, default=15)
    args = parser.parse_args()

    cfg = load_config(args.config)
    log = FeedbackLog(cfg["paths"]["feedback_db"])

    stats = log.stats()
    print("\nUSAGE")
    for key, value in stats.items():
        print(f"  {key:16s} {value}")

    total = stats["queries"]
    if total:
        low = stats["low_confidence"] / total * 100
        clicked = stats["clicks"] / total
        print(f"\n  {low:.0f}% of queries were out-of-corpus")
        print(f"  {clicked:.2f} citation clicks per query")
        if stats["votes_up"] + stats["votes_down"]:
            up = stats["votes_up"] / (stats["votes_up"] + stats["votes_down"]) * 100
            print(f"  {up:.0f}% of votes were positive")

    data = log.recent_failures(limit=200)
    section("ERRORS - these are bugs", data["errors"],
            ["ts", "endpoint", "error_type", "message"], args.limit)
    section("DOWNVOTED - the answer was wrong", data["downvoted"],
            ["ts", "mode", "query", "comment"], args.limit)
    section("OUT OF CORPUS - what people want that you have not ingested",
            data["out_of_corpus"], ["ts", "mode", "query", "mean_top_score"], args.limit)
    section("CONFIDENT BUT NO CLICKS - results may look irrelevant",
            data["never_clicked"], ["ts", "mode", "query"], args.limit)
    section("SLOWEST", data["slowest"], ["latency_ms", "mode", "query"], args.limit)

    if data["out_of_corpus"]:
        words = collections.Counter()
        for row in data["out_of_corpus"]:
            for word in str(row.get("query", "")).lower().split():
                if len(word) > 4:
                    words[word] += 1
        common = words.most_common(10)
        if common:
            print("\nMost common terms in out-of-corpus queries - candidate ingest topics:")
            print("  " + ", ".join(f"{w} ({n})" for w, n in common))
    print()


if __name__ == "__main__":
    main()