Spaces:
Sleeping
Sleeping
feat: the acceptance criteria as a runnable report
Browse filesEight criteria, each a query against the shipped databases, so the run's
outcome is checked rather than asserted and stays re-checkable after every
future ingest.
Three of them permit a shortfall provided it is enumerated, so each prints the
exact ticker, period and concept. A filer that does not tag a concept is a
fact about the filer; a quietly lower number is not a result.
scripts/data_foundation_report.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The data-foundation acceptance criteria, as queries rather than assertions.
|
| 2 |
+
|
| 3 |
+
Spec: docs/superpowers/specs/2026-09-01-data-foundation-design.md
|
| 4 |
+
|
| 5 |
+
Three criteria deliberately permit a shortfall provided it is enumerated. A
|
| 6 |
+
filer that does not tag a concept is a fact about the filer, and recording it
|
| 7 |
+
is the correct outcome; reporting a quietly lower number is not. So every
|
| 8 |
+
criterion that can fall short prints the exact ticker, period and concept.
|
| 9 |
+
|
| 10 |
+
Usage::
|
| 11 |
+
|
| 12 |
+
python -m scripts.data_foundation_report
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import sqlite3
|
| 18 |
+
import sys
|
| 19 |
+
|
| 20 |
+
DB_PATH = "data/metrics.db"
|
| 21 |
+
|
| 22 |
+
NEW_STATEMENT_LINES = (
|
| 23 |
+
"deferred_revenue", "research_development", "sga", "goodwill",
|
| 24 |
+
"intangibles", "cash_and_st_investments", "debt_current",
|
| 25 |
+
"operating_lease_liabilities",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
CORE_STATEMENT_LINES = (
|
| 29 |
+
"operating_cash_flow", "net_income", "total_assets",
|
| 30 |
+
"receivables", "payables", "cogs",
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
GUIDANCE_FLOOR = 40
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _rows() -> list[dict]:
|
| 37 |
+
conn = sqlite3.connect(DB_PATH)
|
| 38 |
+
conn.row_factory = sqlite3.Row
|
| 39 |
+
try:
|
| 40 |
+
return [dict(r) for r in conn.execute("SELECT * FROM metrics")]
|
| 41 |
+
finally:
|
| 42 |
+
conn.close()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _segment_tickers() -> set[str]:
|
| 46 |
+
conn = sqlite3.connect(DB_PATH)
|
| 47 |
+
try:
|
| 48 |
+
return {r[0] for r in conn.execute("SELECT DISTINCT ticker FROM segments")}
|
| 49 |
+
except sqlite3.OperationalError:
|
| 50 |
+
return set()
|
| 51 |
+
finally:
|
| 52 |
+
conn.close()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def build_report() -> list[dict]:
|
| 56 |
+
rows = _rows()
|
| 57 |
+
total = len(rows)
|
| 58 |
+
out: list[dict] = []
|
| 59 |
+
|
| 60 |
+
legacy = [f"{r['ticker']} {r['period']}" for r in rows
|
| 61 |
+
if r.get("data_quality_status") == "LEGACY_UNVERIFIED"]
|
| 62 |
+
out.append({
|
| 63 |
+
"criterion": "no_legacy_unverified_rows", "target": "0",
|
| 64 |
+
"actual": str(len(legacy)), "passed": not legacy, "detail": legacy,
|
| 65 |
+
})
|
| 66 |
+
|
| 67 |
+
no_url = [f"{r['ticker']} {r['period']}" for r in rows if not r.get("source_url")]
|
| 68 |
+
out.append({
|
| 69 |
+
"criterion": "every_row_carries_source_url", "target": "0 missing",
|
| 70 |
+
"actual": str(len(no_url)), "passed": not no_url, "detail": no_url,
|
| 71 |
+
})
|
| 72 |
+
|
| 73 |
+
parsed = [r for r in rows if r.get("guidance_status") == "PARSED"]
|
| 74 |
+
silent = [f"{r['ticker']} {r['period']}" for r in rows
|
| 75 |
+
if r.get("guidance_status") == "NO_GUIDANCE_ISSUED"]
|
| 76 |
+
errored = [f"{r['ticker']} {r['period']}" for r in rows
|
| 77 |
+
if r.get("guidance_status") == "PARSE_ERROR"]
|
| 78 |
+
unquoted = [f"{r['ticker']} {r['period']}" for r in parsed
|
| 79 |
+
if not r.get("guidance_source_quote")]
|
| 80 |
+
out.append({
|
| 81 |
+
"criterion": "guidance_parsed_above_floor", "target": f">= {GUIDANCE_FLOOR}",
|
| 82 |
+
"actual": str(len(parsed)), "passed": len(parsed) >= GUIDANCE_FLOOR,
|
| 83 |
+
"detail": [f"no guidance issued: {len(silent)}", f"parse errors: {len(errored)}"]
|
| 84 |
+
+ errored,
|
| 85 |
+
})
|
| 86 |
+
out.append({
|
| 87 |
+
"criterion": "every_parsed_guidance_has_a_quote", "target": "0 unquoted",
|
| 88 |
+
"actual": str(len(unquoted)), "passed": not unquoted, "detail": unquoted,
|
| 89 |
+
})
|
| 90 |
+
|
| 91 |
+
core_gaps = [
|
| 92 |
+
f"{r['ticker']} {r['period']} {line}"
|
| 93 |
+
for r in rows for line in CORE_STATEMENT_LINES if r.get(line) is None
|
| 94 |
+
]
|
| 95 |
+
out.append({
|
| 96 |
+
"criterion": "core_statement_lines_populated", "target": f"{total} rows each",
|
| 97 |
+
"actual": f"{len(core_gaps)} gaps", "passed": not core_gaps, "detail": core_gaps,
|
| 98 |
+
})
|
| 99 |
+
|
| 100 |
+
new_gaps = [
|
| 101 |
+
f"{r['ticker']} {r['period']} {line}"
|
| 102 |
+
for r in rows for line in NEW_STATEMENT_LINES if r.get(line) is None
|
| 103 |
+
]
|
| 104 |
+
out.append({
|
| 105 |
+
"criterion": "new_statement_lines_populated",
|
| 106 |
+
"target": "populated wherever the filer tags the concept",
|
| 107 |
+
"actual": f"{len(new_gaps)} absences", "passed": True, "detail": new_gaps,
|
| 108 |
+
})
|
| 109 |
+
|
| 110 |
+
fallback = []
|
| 111 |
+
for r in rows:
|
| 112 |
+
try:
|
| 113 |
+
warnings = json.loads(r.get("quality_warnings") or "[]")
|
| 114 |
+
except (TypeError, ValueError):
|
| 115 |
+
warnings = []
|
| 116 |
+
for w in warnings:
|
| 117 |
+
if "fallback_non_sec" in w:
|
| 118 |
+
fallback.append(f"{r['ticker']} {r['period']} {w}")
|
| 119 |
+
out.append({
|
| 120 |
+
"criterion": "no_fallback_non_sec", "target": "0",
|
| 121 |
+
"actual": str(len(fallback)), "passed": not fallback, "detail": fallback,
|
| 122 |
+
})
|
| 123 |
+
|
| 124 |
+
covered = {r["ticker"] for r in rows}
|
| 125 |
+
with_segments = _segment_tickers()
|
| 126 |
+
missing = sorted(covered - with_segments)
|
| 127 |
+
out.append({
|
| 128 |
+
"criterion": "segments_on_every_covered_ticker",
|
| 129 |
+
"target": f"{len(covered)} tickers",
|
| 130 |
+
"actual": str(len(with_segments)), "passed": not missing, "detail": missing,
|
| 131 |
+
})
|
| 132 |
+
|
| 133 |
+
return out
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def main() -> None:
|
| 137 |
+
report = build_report()
|
| 138 |
+
for r in report:
|
| 139 |
+
mark = "PASS" if r["passed"] else "FAIL"
|
| 140 |
+
print(f"[{mark}] {r['criterion']}: {r['actual']} (target {r['target']})")
|
| 141 |
+
for line in r["detail"][:40]:
|
| 142 |
+
print(f" {line}")
|
| 143 |
+
if len(r["detail"]) > 40:
|
| 144 |
+
print(f" ... and {len(r['detail']) - 40} more")
|
| 145 |
+
failed = [r for r in report if not r["passed"]]
|
| 146 |
+
sys.exit(1 if failed else 0)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
if __name__ == "__main__":
|
| 150 |
+
main()
|
tests/test_data_foundation_report.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def _seed(metrics_db, rows):
|
| 2 |
+
metrics_db.init_db()
|
| 3 |
+
for row in rows:
|
| 4 |
+
metrics_db.upsert_metrics(row)
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def test_legacy_rows_fail_the_provenance_criterion(tmp_path, monkeypatch):
|
| 8 |
+
from storage import metrics_db
|
| 9 |
+
|
| 10 |
+
monkeypatch.setattr(metrics_db, "DB_PATH", tmp_path / "metrics.db")
|
| 11 |
+
_seed(metrics_db, [{
|
| 12 |
+
"ticker": "AAPL", "period": "Q12026",
|
| 13 |
+
"data_quality_status": "LEGACY_UNVERIFIED", "source_url": None,
|
| 14 |
+
}])
|
| 15 |
+
|
| 16 |
+
import scripts.data_foundation_report as rep
|
| 17 |
+
|
| 18 |
+
monkeypatch.setattr(rep, "DB_PATH", tmp_path / "metrics.db")
|
| 19 |
+
results = {r["criterion"]: r for r in rep.build_report()}
|
| 20 |
+
|
| 21 |
+
assert results["no_legacy_unverified_rows"]["passed"] is False
|
| 22 |
+
assert results["every_row_carries_source_url"]["passed"] is False
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def test_clean_rows_pass_and_shortfalls_are_enumerated(tmp_path, monkeypatch):
|
| 26 |
+
from storage import metrics_db
|
| 27 |
+
|
| 28 |
+
monkeypatch.setattr(metrics_db, "DB_PATH", tmp_path / "metrics.db")
|
| 29 |
+
_seed(metrics_db, [{
|
| 30 |
+
"ticker": "AAPL", "period": "Q12026",
|
| 31 |
+
"data_quality_status": "VERIFIED",
|
| 32 |
+
"source_url": "https://www.sec.gov/Archives/edgar/data/320193/x/aapl.htm",
|
| 33 |
+
"quality_warnings": [],
|
| 34 |
+
"guidance_status": "NO_GUIDANCE_ISSUED",
|
| 35 |
+
"goodwill": None,
|
| 36 |
+
}])
|
| 37 |
+
|
| 38 |
+
import scripts.data_foundation_report as rep
|
| 39 |
+
|
| 40 |
+
monkeypatch.setattr(rep, "DB_PATH", tmp_path / "metrics.db")
|
| 41 |
+
results = {r["criterion"]: r for r in rep.build_report()}
|
| 42 |
+
|
| 43 |
+
assert results["no_legacy_unverified_rows"]["passed"] is True
|
| 44 |
+
assert results["every_row_carries_source_url"]["passed"] is True
|
| 45 |
+
# An absent line is enumerated, never silently counted as a pass.
|
| 46 |
+
assert "AAPL Q12026" in " ".join(results["new_statement_lines_populated"]["detail"])
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def test_fallback_rows_are_named_not_just_counted(tmp_path, monkeypatch):
|
| 50 |
+
from storage import metrics_db
|
| 51 |
+
|
| 52 |
+
monkeypatch.setattr(metrics_db, "DB_PATH", tmp_path / "metrics.db")
|
| 53 |
+
_seed(metrics_db, [{
|
| 54 |
+
"ticker": "GOOGL", "period": "FY2025",
|
| 55 |
+
"data_quality_status": "CHECK_REQUIRED",
|
| 56 |
+
"source_url": "https://www.sec.gov/Archives/edgar/data/1652044/x/goog.htm",
|
| 57 |
+
"quality_warnings": ["gross_margin:fallback_non_sec"],
|
| 58 |
+
}])
|
| 59 |
+
|
| 60 |
+
import scripts.data_foundation_report as rep
|
| 61 |
+
|
| 62 |
+
monkeypatch.setattr(rep, "DB_PATH", tmp_path / "metrics.db")
|
| 63 |
+
results = {r["criterion"]: r for r in rep.build_report()}
|
| 64 |
+
|
| 65 |
+
detail = " ".join(results["no_fallback_non_sec"]["detail"])
|
| 66 |
+
assert "GOOGL FY2025" in detail
|
| 67 |
+
assert "gross_margin" in detail
|