Spaces:
Sleeping
fix: survive an LFS pointer at data/metrics.db in the daily job
Browse files.gitattributes tracks data/*.db under Git LFS and actions/checkout@v4
defaults to lfs: false, so on the CI runner data/metrics.db is a ~130-byte
pointer: DB_PATH.exists() is True and sqlite3 then raises
"file is not a database". The daily consensus job would traceback every
single day, on the one archive whose gaps cannot be backfilled from any
public source.
Both halves fixed:
- the workflow pulls only data/metrics.db from LFS. `lfs: true` on the
checkout would fetch ~100 MB of LFS objects daily (80 MB of it
data/chroma/chroma.sqlite3, unread by this job) and exhaust GitHub's LFS
bandwidth quota within weeks — and an exhausted quota reinstates the
pointer, i.e. the bug.
- list_tickers() catches sqlite3.DatabaseError and returns [], so the
caller's existing "no covered tickers found" path fires with a readable
message and exit code 1 instead of an unhandled traceback.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
- .github/workflows/snapshot-consensus.yml +16 -0
- storage/metrics_db.py +19 -7
- tests/test_metrics_db.py +22 -0
|
@@ -20,6 +20,22 @@ jobs:
|
|
| 20 |
steps:
|
| 21 |
- uses: actions/checkout@v4
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
- uses: actions/setup-python@v5
|
| 24 |
with:
|
| 25 |
python-version: "3.11"
|
|
|
|
| 20 |
steps:
|
| 21 |
- uses: actions/checkout@v4
|
| 22 |
|
| 23 |
+
# data/metrics.db is tracked by Git LFS (.gitattributes: `data/*.db`),
|
| 24 |
+
# and actions/checkout@v4 defaults to `lfs: false` — so without this the
|
| 25 |
+
# file on the runner is a ~130-byte pointer, storage/metrics_db.py opens
|
| 26 |
+
# it, and sqlite3 raises "file is not a database". The job would fail
|
| 27 |
+
# every day, on the one archive whose missed days cannot be recovered
|
| 28 |
+
# from any public source.
|
| 29 |
+
#
|
| 30 |
+
# This pulls ONLY metrics.db (~1 MB) rather than setting `lfs: true` on
|
| 31 |
+
# the checkout, which would fetch every LFS object in the repo — ~100 MB,
|
| 32 |
+
# 80 MB of it data/chroma/chroma.sqlite3, none of which this job reads.
|
| 33 |
+
# At a daily cadence that would exhaust GitHub's LFS bandwidth quota
|
| 34 |
+
# within a couple of weeks, and an exhausted quota fails the LFS fetch,
|
| 35 |
+
# which puts the pointer file straight back. Fetch what the job needs.
|
| 36 |
+
- name: Fetch the metrics DB from LFS
|
| 37 |
+
run: git lfs pull --include="data/metrics.db"
|
| 38 |
+
|
| 39 |
- uses: actions/setup-python@v5
|
| 40 |
with:
|
| 41 |
python-version: "3.11"
|
|
@@ -289,16 +289,28 @@ def list_tickers() -> list[str]:
|
|
| 289 |
This is the app's de facto coverage universe: there is no explicit
|
| 290 |
coverage table, and scripts/warm_runtime_cache.py and deploy_check.py
|
| 291 |
have both derived it with this same query.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
"""
|
| 293 |
if not DB_PATH.exists():
|
| 294 |
return []
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
|
|
|
|
|
|
|
|
|
| 302 |
|
| 303 |
|
| 304 |
def get_all_metrics(ticker: str) -> list[dict]:
|
|
|
|
| 289 |
This is the app's de facto coverage universe: there is no explicit
|
| 290 |
coverage table, and scripts/warm_runtime_cache.py and deploy_check.py
|
| 291 |
have both derived it with this same query.
|
| 292 |
+
|
| 293 |
+
A path that exists but is not a SQLite database yields [] rather than
|
| 294 |
+
raising. That is not hypothetical: `.gitattributes` tracks `data/*.db`
|
| 295 |
+
under Git LFS, and `actions/checkout@v4` defaults to `lfs: false`, so on
|
| 296 |
+
a CI runner without an LFS pull this file is a ~130-byte pointer and
|
| 297 |
+
`sqlite3.connect(...).execute(...)` raises
|
| 298 |
+
`sqlite3.DatabaseError: file is not a database`. Callers already handle
|
| 299 |
+
"no covered tickers" with a readable message; an unhandled traceback
|
| 300 |
+
would just be a noisier way of saying the same thing.
|
| 301 |
"""
|
| 302 |
if not DB_PATH.exists():
|
| 303 |
return []
|
| 304 |
+
try:
|
| 305 |
+
with sqlite3.connect(DB_PATH) as conn:
|
| 306 |
+
return [
|
| 307 |
+
row[0]
|
| 308 |
+
for row in conn.execute(
|
| 309 |
+
"SELECT DISTINCT ticker FROM metrics ORDER BY ticker"
|
| 310 |
+
)
|
| 311 |
+
]
|
| 312 |
+
except sqlite3.DatabaseError:
|
| 313 |
+
return []
|
| 314 |
|
| 315 |
|
| 316 |
def get_all_metrics(ticker: str) -> list[dict]:
|
|
@@ -168,3 +168,25 @@ def test_list_tickers_on_a_missing_database_returns_empty(tmp_path, monkeypatch)
|
|
| 168 |
import storage.metrics_db as m
|
| 169 |
monkeypatch.setattr(m, "DB_PATH", tmp_path / "absent.db")
|
| 170 |
assert m.list_tickers() == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
import storage.metrics_db as m
|
| 169 |
monkeypatch.setattr(m, "DB_PATH", tmp_path / "absent.db")
|
| 170 |
assert m.list_tickers() == []
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def test_list_tickers_on_a_non_database_file_returns_empty(tmp_path, monkeypatch):
|
| 174 |
+
"""A Git LFS pointer is a text file that exists but is not SQLite.
|
| 175 |
+
|
| 176 |
+
`.gitattributes` tracks `data/*.db` under LFS, and `actions/checkout@v4`
|
| 177 |
+
defaults to `lfs: false`, so on a CI runner without an LFS pull the path
|
| 178 |
+
exists and holds a ~130-byte pointer. Opening it raises
|
| 179 |
+
`sqlite3.DatabaseError: file is not a database`. list_tickers must degrade
|
| 180 |
+
to "no coverage" so the caller prints its own message and exits 1, rather
|
| 181 |
+
than dying with an unhandled traceback.
|
| 182 |
+
"""
|
| 183 |
+
import storage.metrics_db as m
|
| 184 |
+
pointer = tmp_path / "metrics.db"
|
| 185 |
+
pointer.write_text(
|
| 186 |
+
"version https://git-lfs.github.com/spec/v1\n"
|
| 187 |
+
"oid sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\n"
|
| 188 |
+
"size 4194304\n",
|
| 189 |
+
encoding="utf-8",
|
| 190 |
+
)
|
| 191 |
+
monkeypatch.setattr(m, "DB_PATH", pointer)
|
| 192 |
+
assert m.list_tickers() == []
|