"""Day-4 evaluation set — the questions that make the system *provable*. Four tiers, each probing a distinct failure mode of an agentic RAG system: factual → sql route. One exact top-level figure. We assert the number appears in the answer. Ground truth is read from our own `financial_facts` table (correct-by-construction), so this tier is really a regression check: does the agent route to sql and report the figure *faithfully*, or hallucinate / mis-label it (the exact bug that survived Day 2)? narrative → vector route. Qualitative content. No numeric ground truth; graded by the LLM judge on faithfulness + answer relevance, and by a deterministic check that the answer actually cites [N]. multihop → both + calculator. A figure that must be *computed* (YoY growth, margin) or a cross-company comparison. Tests retrieval + tool use + arithmetic end to end. honesty → unanswerable from the corpus (a company/year we don't have, or a forward-looking number). The agent must DECLINE, not fabricate — the single most important behaviour for a finance assistant. Ground truth for the numeric tiers is a `gt` callable resolved lazily against DuckDB, so this module imports without touching the DB and the numbers can never drift from what the agent can actually query. """ from __future__ import annotations from collections.abc import Callable from dataclasses import dataclass from finrag.ingestion.facts import query # ── Tiers ──────────────────────────────────────────────────────────────── FACTUAL = "factual" NARRATIVE = "narrative" MULTIHOP = "multihop" HONESTY = "honesty" @dataclass(frozen=True) class EvalCase: id: str tier: str question: str # Soft expectation — the plan node's route. We record match/mismatch but a # mismatch isn't a hard failure (the agent can still answer correctly via a # different path). route_expected: str | None = None # Numeric ground truth, resolved lazily from the DB at run time. gt: Callable[[], float] | None = None gt_kind: str = "currency" # currency | per_share | percent # Relative tolerance for currency/per_share; for percent it's the absolute # percentage-point floor (combined with a 5% relative band in the matcher). tol: float = 0.015 # For name-style answers ("which company…") and any case where a substring # must appear (e.g. the right company name). expect_substring: str | None = None # Honesty tier: the agent must refuse / say it's not in the corpus. must_decline: bool = False # ── Ground-truth helper ────────────────────────────────────────────────── def _v(ticker: str, line_item: str, fy: int, unit: str = "USD") -> float: """The single authoritative value for a (ticker, line_item, year) from financial_facts — the same table sql_query reads. LIMIT 1 because each of our three filers resolves to one surviving GAAP concept per annual period (see facts.py dedup); ordering by value desc is a stable tie-break.""" rows = query( """ SELECT value FROM financial_facts WHERE ticker = ? AND line_item = ? AND fiscal_year = ? AND fiscal_period = 'FY' AND unit = ? ORDER BY value DESC LIMIT 1 """, [ticker, line_item, fy, unit], ) if not rows: raise LookupError(f"no fact for {ticker} {line_item} FY{fy} ({unit})") return float(rows[0]["value"]) def _yoy(ticker: str, line_item: str, y0: int, y1: int) -> float: """Year-over-year percent growth from y0 to y1.""" a, b = _v(ticker, line_item, y0), _v(ticker, line_item, y1) return (b - a) / a * 100.0 def _margin(ticker: str, part: str, whole: str, fy: int) -> float: """A margin in percent (e.g. gross_profit / revenue).""" return _v(ticker, part, fy) / _v(ticker, whole, fy) * 100.0 # ── The set ────────────────────────────────────────────────────────────── CASES: list[EvalCase] = [ # ─────────── FACTUAL (sql route, exact figure) ─────────── EvalCase("f01", FACTUAL, "What was Apple's net income in fiscal 2023?", route_expected="sql", gt=lambda: _v("AAPL", "net_income", 2023)), EvalCase("f02", FACTUAL, "What was Apple's total revenue in fiscal 2024?", route_expected="sql", gt=lambda: _v("AAPL", "revenue", 2024)), EvalCase("f03", FACTUAL, "How much did Tesla spend on research and development in fiscal 2023?", route_expected="sql", gt=lambda: _v("TSLA", "rd_expense", 2023)), EvalCase("f04", FACTUAL, "What were JPMorgan's total assets at the end of fiscal 2023?", route_expected="sql", gt=lambda: _v("JPM", "total_assets", 2023)), EvalCase("f05", FACTUAL, "What was Apple's operating income in fiscal 2022?", route_expected="sql", gt=lambda: _v("AAPL", "operating_income", 2022)), EvalCase("f06", FACTUAL, "What was Tesla's total revenue in fiscal 2024?", route_expected="sql", gt=lambda: _v("TSLA", "revenue", 2024)), EvalCase("f07", FACTUAL, "What was JPMorgan's net income in fiscal 2024?", route_expected="sql", gt=lambda: _v("JPM", "net_income", 2024)), EvalCase("f08", FACTUAL, "What was Apple's gross profit in fiscal 2023?", route_expected="sql", gt=lambda: _v("AAPL", "gross_profit", 2023)), EvalCase("f09", FACTUAL, "What was Tesla's diluted earnings per share in fiscal 2023?", route_expected="sql", gt=lambda: _v("TSLA", "eps_diluted", 2023, "USD/shares"), gt_kind="per_share"), EvalCase("f10", FACTUAL, "What was JPMorgan's net interest income in fiscal 2024?", route_expected="sql", gt=lambda: _v("JPM", "net_interest_income", 2024)), # ─────────── NARRATIVE (vector route, faithfulness + citation) ─────────── EvalCase("n01", NARRATIVE, "How does Apple describe the risks to its supply chain in its 10-K?", route_expected="vector"), EvalCase("n02", NARRATIVE, "What does Tesla say about competition in the electric-vehicle market?", route_expected="vector"), EvalCase("n03", NARRATIVE, "How does JPMorgan describe credit risk in its filing?", route_expected="vector"), EvalCase("n04", NARRATIVE, "How does Apple characterize its Services business and what drives its growth?", route_expected="vector"), EvalCase("n05", NARRATIVE, "What risks does Tesla cite related to its dependence on key personnel?", route_expected="vector"), EvalCase("n06", NARRATIVE, "How does Apple describe foreign-currency exchange-rate risk?", route_expected="vector"), EvalCase("n07", NARRATIVE, "What does Tesla say about risks in ramping production and manufacturing?", route_expected="vector"), EvalCase("n08", NARRATIVE, "How does JPMorgan describe the regulatory and capital requirements it faces?", route_expected="vector"), # ─────────── MULTIHOP (sql + calculator; route left soft) ─────────── # route_expected is None: each of these is answerable from structured facts # plus arithmetic, so the router legitimately picks `sql` over `both`. The # tier tests multi-step reasoning (fetch figure(s) → compute), not the route. EvalCase("m01", MULTIHOP, "By what percentage did Apple's net income change from fiscal 2022 to fiscal 2023?", gt=lambda: _yoy("AAPL", "net_income", 2022, 2023), gt_kind="percent"), EvalCase("m02", MULTIHOP, "By what percentage did Tesla's revenue grow from fiscal 2023 to fiscal 2024?", gt=lambda: _yoy("TSLA", "revenue", 2023, 2024), gt_kind="percent"), EvalCase("m03", MULTIHOP, "What was Apple's gross margin in fiscal 2023?", gt=lambda: _margin("AAPL", "gross_profit", "revenue", 2023), gt_kind="percent"), EvalCase("m04", MULTIHOP, "What was Tesla's net profit margin in fiscal 2023?", gt=lambda: _margin("TSLA", "net_income", "revenue", 2023), gt_kind="percent"), EvalCase("m05", MULTIHOP, "What was Apple's operating margin in fiscal 2024?", gt=lambda: _margin("AAPL", "operating_income", "revenue", 2024), gt_kind="percent"), EvalCase("m06", MULTIHOP, "Which of the three companies had the highest net income in fiscal 2023?", expect_substring="Apple"), EvalCase("m07", MULTIHOP, "By what percentage did JPMorgan's net income change from fiscal 2023 to fiscal 2024?", gt=lambda: _yoy("JPM", "net_income", 2023, 2024), gt_kind="percent"), # ─────────── HONESTY (must decline — outside the corpus) ─────────── EvalCase("h01", HONESTY, "What was Microsoft's net income in fiscal 2023?", must_decline=True), EvalCase("h02", HONESTY, "What was Apple's total revenue in fiscal 2019?", must_decline=True), # outside our FY2022–2024 span EvalCase("h03", HONESTY, "How many employees does Amazon have according to its 10-K?", must_decline=True), # company not in corpus EvalCase("h04", HONESTY, "What will Tesla's revenue be in fiscal 2026?", must_decline=True), # forward-looking, not in any filing EvalCase("h05", HONESTY, "What was Google's operating margin in fiscal 2023?", must_decline=True), # company not in corpus ] def cases_for(tier: str | None = None) -> list[EvalCase]: return [c for c in CASES if tier is None or c.tier == tier] def one_per_tier() -> list[EvalCase]: """A 4-case smoke subset — first case of each tier. Used to validate the harness end to end before spending on a full run, and as the Gemini A/B subset that fits the free-tier daily quota.""" out: list[EvalCase] = [] for tier in (FACTUAL, NARRATIVE, MULTIHOP, HONESTY): out.append(cases_for(tier)[0]) return out