Viney commited on
Commit
2f5da7d
·
1 Parent(s): a21428e

feat: the acceptance criteria as a runnable report

Browse files

Eight criteria, each a query against the shipped databases, so the run's
outcome is checked rather than asserted and stays re-checkable after every
future ingest.

Three of them permit a shortfall provided it is enumerated, so each prints the
exact ticker, period and concept. A filer that does not tag a concept is a
fact about the filer; a quietly lower number is not a result.

scripts/data_foundation_report.py ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The data-foundation acceptance criteria, as queries rather than assertions.
2
+
3
+ Spec: docs/superpowers/specs/2026-09-01-data-foundation-design.md
4
+
5
+ Three criteria deliberately permit a shortfall provided it is enumerated. A
6
+ filer that does not tag a concept is a fact about the filer, and recording it
7
+ is the correct outcome; reporting a quietly lower number is not. So every
8
+ criterion that can fall short prints the exact ticker, period and concept.
9
+
10
+ Usage::
11
+
12
+ python -m scripts.data_foundation_report
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import sqlite3
18
+ import sys
19
+
20
+ DB_PATH = "data/metrics.db"
21
+
22
+ NEW_STATEMENT_LINES = (
23
+ "deferred_revenue", "research_development", "sga", "goodwill",
24
+ "intangibles", "cash_and_st_investments", "debt_current",
25
+ "operating_lease_liabilities",
26
+ )
27
+
28
+ CORE_STATEMENT_LINES = (
29
+ "operating_cash_flow", "net_income", "total_assets",
30
+ "receivables", "payables", "cogs",
31
+ )
32
+
33
+ GUIDANCE_FLOOR = 40
34
+
35
+
36
+ def _rows() -> list[dict]:
37
+ conn = sqlite3.connect(DB_PATH)
38
+ conn.row_factory = sqlite3.Row
39
+ try:
40
+ return [dict(r) for r in conn.execute("SELECT * FROM metrics")]
41
+ finally:
42
+ conn.close()
43
+
44
+
45
+ def _segment_tickers() -> set[str]:
46
+ conn = sqlite3.connect(DB_PATH)
47
+ try:
48
+ return {r[0] for r in conn.execute("SELECT DISTINCT ticker FROM segments")}
49
+ except sqlite3.OperationalError:
50
+ return set()
51
+ finally:
52
+ conn.close()
53
+
54
+
55
+ def build_report() -> list[dict]:
56
+ rows = _rows()
57
+ total = len(rows)
58
+ out: list[dict] = []
59
+
60
+ legacy = [f"{r['ticker']} {r['period']}" for r in rows
61
+ if r.get("data_quality_status") == "LEGACY_UNVERIFIED"]
62
+ out.append({
63
+ "criterion": "no_legacy_unverified_rows", "target": "0",
64
+ "actual": str(len(legacy)), "passed": not legacy, "detail": legacy,
65
+ })
66
+
67
+ no_url = [f"{r['ticker']} {r['period']}" for r in rows if not r.get("source_url")]
68
+ out.append({
69
+ "criterion": "every_row_carries_source_url", "target": "0 missing",
70
+ "actual": str(len(no_url)), "passed": not no_url, "detail": no_url,
71
+ })
72
+
73
+ parsed = [r for r in rows if r.get("guidance_status") == "PARSED"]
74
+ silent = [f"{r['ticker']} {r['period']}" for r in rows
75
+ if r.get("guidance_status") == "NO_GUIDANCE_ISSUED"]
76
+ errored = [f"{r['ticker']} {r['period']}" for r in rows
77
+ if r.get("guidance_status") == "PARSE_ERROR"]
78
+ unquoted = [f"{r['ticker']} {r['period']}" for r in parsed
79
+ if not r.get("guidance_source_quote")]
80
+ out.append({
81
+ "criterion": "guidance_parsed_above_floor", "target": f">= {GUIDANCE_FLOOR}",
82
+ "actual": str(len(parsed)), "passed": len(parsed) >= GUIDANCE_FLOOR,
83
+ "detail": [f"no guidance issued: {len(silent)}", f"parse errors: {len(errored)}"]
84
+ + errored,
85
+ })
86
+ out.append({
87
+ "criterion": "every_parsed_guidance_has_a_quote", "target": "0 unquoted",
88
+ "actual": str(len(unquoted)), "passed": not unquoted, "detail": unquoted,
89
+ })
90
+
91
+ core_gaps = [
92
+ f"{r['ticker']} {r['period']} {line}"
93
+ for r in rows for line in CORE_STATEMENT_LINES if r.get(line) is None
94
+ ]
95
+ out.append({
96
+ "criterion": "core_statement_lines_populated", "target": f"{total} rows each",
97
+ "actual": f"{len(core_gaps)} gaps", "passed": not core_gaps, "detail": core_gaps,
98
+ })
99
+
100
+ new_gaps = [
101
+ f"{r['ticker']} {r['period']} {line}"
102
+ for r in rows for line in NEW_STATEMENT_LINES if r.get(line) is None
103
+ ]
104
+ out.append({
105
+ "criterion": "new_statement_lines_populated",
106
+ "target": "populated wherever the filer tags the concept",
107
+ "actual": f"{len(new_gaps)} absences", "passed": True, "detail": new_gaps,
108
+ })
109
+
110
+ fallback = []
111
+ for r in rows:
112
+ try:
113
+ warnings = json.loads(r.get("quality_warnings") or "[]")
114
+ except (TypeError, ValueError):
115
+ warnings = []
116
+ for w in warnings:
117
+ if "fallback_non_sec" in w:
118
+ fallback.append(f"{r['ticker']} {r['period']} {w}")
119
+ out.append({
120
+ "criterion": "no_fallback_non_sec", "target": "0",
121
+ "actual": str(len(fallback)), "passed": not fallback, "detail": fallback,
122
+ })
123
+
124
+ covered = {r["ticker"] for r in rows}
125
+ with_segments = _segment_tickers()
126
+ missing = sorted(covered - with_segments)
127
+ out.append({
128
+ "criterion": "segments_on_every_covered_ticker",
129
+ "target": f"{len(covered)} tickers",
130
+ "actual": str(len(with_segments)), "passed": not missing, "detail": missing,
131
+ })
132
+
133
+ return out
134
+
135
+
136
+ def main() -> None:
137
+ report = build_report()
138
+ for r in report:
139
+ mark = "PASS" if r["passed"] else "FAIL"
140
+ print(f"[{mark}] {r['criterion']}: {r['actual']} (target {r['target']})")
141
+ for line in r["detail"][:40]:
142
+ print(f" {line}")
143
+ if len(r["detail"]) > 40:
144
+ print(f" ... and {len(r['detail']) - 40} more")
145
+ failed = [r for r in report if not r["passed"]]
146
+ sys.exit(1 if failed else 0)
147
+
148
+
149
+ if __name__ == "__main__":
150
+ main()
tests/test_data_foundation_report.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def _seed(metrics_db, rows):
2
+ metrics_db.init_db()
3
+ for row in rows:
4
+ metrics_db.upsert_metrics(row)
5
+
6
+
7
+ def test_legacy_rows_fail_the_provenance_criterion(tmp_path, monkeypatch):
8
+ from storage import metrics_db
9
+
10
+ monkeypatch.setattr(metrics_db, "DB_PATH", tmp_path / "metrics.db")
11
+ _seed(metrics_db, [{
12
+ "ticker": "AAPL", "period": "Q12026",
13
+ "data_quality_status": "LEGACY_UNVERIFIED", "source_url": None,
14
+ }])
15
+
16
+ import scripts.data_foundation_report as rep
17
+
18
+ monkeypatch.setattr(rep, "DB_PATH", tmp_path / "metrics.db")
19
+ results = {r["criterion"]: r for r in rep.build_report()}
20
+
21
+ assert results["no_legacy_unverified_rows"]["passed"] is False
22
+ assert results["every_row_carries_source_url"]["passed"] is False
23
+
24
+
25
+ def test_clean_rows_pass_and_shortfalls_are_enumerated(tmp_path, monkeypatch):
26
+ from storage import metrics_db
27
+
28
+ monkeypatch.setattr(metrics_db, "DB_PATH", tmp_path / "metrics.db")
29
+ _seed(metrics_db, [{
30
+ "ticker": "AAPL", "period": "Q12026",
31
+ "data_quality_status": "VERIFIED",
32
+ "source_url": "https://www.sec.gov/Archives/edgar/data/320193/x/aapl.htm",
33
+ "quality_warnings": [],
34
+ "guidance_status": "NO_GUIDANCE_ISSUED",
35
+ "goodwill": None,
36
+ }])
37
+
38
+ import scripts.data_foundation_report as rep
39
+
40
+ monkeypatch.setattr(rep, "DB_PATH", tmp_path / "metrics.db")
41
+ results = {r["criterion"]: r for r in rep.build_report()}
42
+
43
+ assert results["no_legacy_unverified_rows"]["passed"] is True
44
+ assert results["every_row_carries_source_url"]["passed"] is True
45
+ # An absent line is enumerated, never silently counted as a pass.
46
+ assert "AAPL Q12026" in " ".join(results["new_statement_lines_populated"]["detail"])
47
+
48
+
49
+ def test_fallback_rows_are_named_not_just_counted(tmp_path, monkeypatch):
50
+ from storage import metrics_db
51
+
52
+ monkeypatch.setattr(metrics_db, "DB_PATH", tmp_path / "metrics.db")
53
+ _seed(metrics_db, [{
54
+ "ticker": "GOOGL", "period": "FY2025",
55
+ "data_quality_status": "CHECK_REQUIRED",
56
+ "source_url": "https://www.sec.gov/Archives/edgar/data/1652044/x/goog.htm",
57
+ "quality_warnings": ["gross_margin:fallback_non_sec"],
58
+ }])
59
+
60
+ import scripts.data_foundation_report as rep
61
+
62
+ monkeypatch.setattr(rep, "DB_PATH", tmp_path / "metrics.db")
63
+ results = {r["criterion"]: r for r in rep.build_report()}
64
+
65
+ detail = " ".join(results["no_fallback_non_sec"]["detail"])
66
+ assert "GOOGL FY2025" in detail
67
+ assert "gross_margin" in detail