Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download platform/core/do_usage.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 27.3 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/platform/core/do_usage.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/platform/core/do_usage.py
-
curl -L -o do_usage.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/platform/core/do_usage.py
27.3 kB
| """DigitalOcean billing observations, never SQL or a database wake-up probe. | |
| The card uses GETs and memory only. The optional sampler receives storage | |
| callbacks from the host and shares Neon's management-plane store-awake guard. | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import json | |
| import math | |
| import os | |
| import re | |
| import threading | |
| from datetime import datetime, timezone | |
| from urllib.error import HTTPError, URLError | |
| from urllib.request import HTTPRedirectHandler, Request, build_opener | |
| from core.neon_usage import send_telegram, store_is_awake | |
| API_BASE = "https://api.digitalocean.com/v2" | |
| HTTP_TIMEOUT = 5 | |
| DEFAULT_CLUSTER_ID = "0c4049b4-8541-43a7-ab6e-a69c34a8c604" | |
| STORE_KEY = "platform_do_usage_samples" | |
| MAX_SAMPLES = 240 | |
| FIXED_PRICES = {"db-s-1vcpu-1gb": {"usd": 15.15, "source": "R28 fixed monthly price per node"}} | |
| CLUSTER_SOURCE = "DigitalOcean databases API" | |
| BALANCE_SOURCE = "DigitalOcean balance API (daily snapshot, account-wide)" | |
| BILLING_SOURCE = "DigitalOcean billing history API" | |
| class UsageReadError(Exception): | |
| """Fixed, credential-free cause safe to show to an operator.""" | |
| class _NoRedirect(HTTPRedirectHandler): | |
| def redirect_request(self, req, fp, code, msg, headers, newurl): | |
| return None | |
| def _get(path, token): | |
| try: | |
| request = Request(API_BASE + path, headers={"Authorization": "Bearer " + token}, method="GET") | |
| with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response: | |
| payload = json.loads(response.read(1_000_001)) | |
| if not isinstance(payload, dict): | |
| raise ValueError() | |
| return payload | |
| except HTTPError as exc: | |
| raise UsageReadError(f"DigitalOcean API returned HTTP {exc.code}.") from None | |
| except TimeoutError: | |
| raise UsageReadError("DigitalOcean API timed out.") from None | |
| except URLError as exc: | |
| cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached" | |
| raise UsageReadError(f"DigitalOcean API {cause}.") from None | |
| except Exception: | |
| # Do not return the URL, response body or exception text: any can contain credentials. | |
| raise UsageReadError("DigitalOcean API returned an unreadable response.") from None | |
| def _now(): | |
| return datetime.now(timezone.utc).isoformat(timespec="seconds") | |
| def _stamp(value): | |
| try: | |
| parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) | |
| return parsed.astimezone(timezone.utc) if parsed.tzinfo else None | |
| except (ValueError, TypeError): | |
| return None | |
| def _text(value): | |
| if not isinstance(value, str) or not value.strip(): | |
| raise ValueError() | |
| return value.replace("\u2014", "; ").replace("\u2013", " to ")[:512] | |
| def _money(value): | |
| if isinstance(value, bool) or value is None: | |
| raise ValueError() | |
| number = float(value) | |
| if not math.isfinite(number): | |
| raise ValueError() | |
| return number | |
| def read_account(): | |
| """Independently read each API block; allowlist fields to exclude connection secrets.""" | |
| read_at = _now() | |
| cluster = {"available": False, "cause": None, "source": CLUSTER_SOURCE, "read_at": read_at, | |
| **dict.fromkeys(("id", "name", "size", "num_nodes", "region", "status", "version", "storage_size_mib"))} | |
| balance = {"available": False, "cause": None, "source": BALANCE_SOURCE, "read_at": read_at, | |
| **dict.fromkeys(("month_to_date_usage", "account_balance", "month_to_date_balance", "generated_at"))} | |
| billing = {"available": False, "cause": None, "source": BILLING_SOURCE, "read_at": read_at} | |
| price = {"monthly_usd": None, "source": None, "read_at": read_at, "cause": None} | |
| result = {"available": False, "cause": None, "read_at": read_at, "cluster": cluster, | |
| "balance": balance, "billing": billing, "billing_history": [], "fixed_price": price} | |
| token = os.environ.get("AIOS_DO_USAGE_TOKEN") or os.environ.get("DIGITALOCEAN_TOKEN", "") | |
| if not token: | |
| cause = "No DigitalOcean token is configured on this deployment, so usage cannot be read." | |
| result["cause"] = price["cause"] = cause | |
| for block in (cluster, balance, billing): | |
| block["cause"] = cause | |
| return result | |
| selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID) | |
| try: | |
| if not re.fullmatch(r"[a-zA-Z0-9-]{1,128}", selected): | |
| raise UsageReadError("AIOS_DO_CLUSTER_ID is invalid; the cluster could not be read.") | |
| database = _get("/databases/" + selected, token).get("database") | |
| if not isinstance(database, dict) or database.get("id") != selected: | |
| raise ValueError() | |
| fields = {key: _text(database.get(key)) for key in ("id", "name", "size", "region", "status", "version")} | |
| nodes = database.get("num_nodes") | |
| if type(nodes) is not int or nodes < 1: | |
| raise ValueError() | |
| storage = database.get("storage_size_mib") | |
| fields.update(num_nodes=nodes, storage_size_mib=storage if type(storage) is int and storage >= 0 else None) | |
| cluster.update(fields, available=True, read_at=_now()) | |
| quoted = FIXED_PRICES.get(fields["size"]) | |
| if quoted: | |
| price.update(monthly_usd=round(quoted["usd"] * nodes, 2), source=quoted["source"], read_at=cluster["read_at"]) | |
| else: | |
| price["cause"] = f"Fixed price UNKNOWN for size {fields['size']}; no fixed price is configured." | |
| except UsageReadError as exc: | |
| cluster["cause"] = str(exc) | |
| except Exception: | |
| cluster["cause"] = "DigitalOcean cluster response is missing valid cluster details." | |
| if not cluster["available"]: | |
| price["cause"] = cluster["cause"] | |
| try: | |
| payload = _get("/customers/my/balance", token) | |
| amounts = {key: _money(payload.get(key)) for key in | |
| ("month_to_date_usage", "account_balance", "month_to_date_balance")} | |
| generated_at = payload.get("generated_at") | |
| if amounts["month_to_date_usage"] < 0 or not _stamp(generated_at): | |
| raise ValueError() | |
| balance.update(amounts, generated_at=generated_at, available=True, read_at=_now()) | |
| except UsageReadError as exc: | |
| balance["cause"] = str(exc) | |
| except Exception: | |
| balance["cause"] = "DigitalOcean balance response is missing valid amounts or its snapshot time." | |
| try: | |
| payload = _get("/customers/my/billing_history", token) | |
| rows = payload.get("billing_history") | |
| # A new account's empty history arrives as the bare list envelope {"links": {}, "meta": {}} | |
| # with the key omitted (read 2026-09-25): an empty history, not an unreadable one. | |
| if rows is None and isinstance(payload, dict) and "meta" in payload: | |
| rows = [] | |
| if not isinstance(rows, list): | |
| raise ValueError() | |
| history = [] | |
| for row in rows: | |
| date = row.get("date") | |
| if not _stamp(date): | |
| raise ValueError() | |
| history.append({"date": date, "description": _text(row.get("description")), | |
| "amount": _money(row.get("amount")), "type": _text(row.get("type"))}) | |
| # Report a bounded page honestly; never follow an API-provided URL with a bearer token. | |
| pages = (payload.get("links") or {}).get("pages") or {} | |
| billing.update(available=True, read_at=_now(), has_more=bool(pages.get("next"))) | |
| result["billing_history"] = history | |
| except UsageReadError as exc: | |
| billing["cause"] = str(exc) | |
| except Exception: | |
| billing["cause"] = "DigitalOcean billing history response could not be read." | |
| result["available"] = any(block["available"] for block in (cluster, balance, billing)) | |
| if not result["available"]: | |
| result["cause"] = " ".join(f"{name}: {block['cause']}" for name, block in | |
| (("Cluster", cluster), ("Balance", balance), ("Billing history", billing))) | |
| return result | |
| def evaluate_alarm(account): | |
| """Account-wide MTD against the sum of the configured cluster's fixed node prices.""" | |
| balance, price = account["balance"], account["fixed_price"] | |
| if not balance["available"]: | |
| return None, "Alert evaluation deferred. " + balance["cause"] | |
| expected = price["monthly_usd"] | |
| if expected is None: | |
| return None, "Alert evaluation deferred. " + price["cause"] | |
| override = os.environ.get("AIOS_DO_ALARM_USD") | |
| try: | |
| threshold = expected if override is None else _money(override) | |
| except (ValueError, TypeError, OverflowError): | |
| return None, "AIOS_DO_ALARM_USD is invalid; alert evaluation is deferred." | |
| if balance["month_to_date_usage"] <= threshold: | |
| return None, None | |
| test = override is not None | |
| cluster = account["cluster"] | |
| message = ("TEST " if test else "") + ( | |
| f"DigitalOcean month-to-date spend USD {balance['month_to_date_usage']:.2f}, " | |
| f"as of {balance['generated_at']} ({balance['source']}; read {balance['read_at']}). " | |
| f"Expected fixed price USD {expected:.2f} a month ({price['source']}; read {price['read_at']}). " | |
| f"Cluster {cluster['name']} ({cluster['size']}, {cluster['num_nodes']} " | |
| f"node{'' if cluster['num_nodes'] == 1 else 's'}, {cluster['region']}).") | |
| if test: | |
| message += f" TEST threshold USD {threshold:.2f} (AIOS_DO_ALARM_USD; read {account['read_at']})." | |
| return {"level": "red", "message": message, "test": test, "threshold": threshold, | |
| "period": _stamp(balance["generated_at"]).strftime("%Y-%m")}, None | |
| _lock = threading.RLock() | |
| _sample_lock = threading.Lock() | |
| _thread = None | |
| _stop = threading.Event() | |
| _samples = [] | |
| _alerted = {} | |
| _loaded = False | |
| _cluster_id = None | |
| _status = {"every_s": 0, "last_at": None, "persisted": None, | |
| "cause": "Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset or zero."} | |
| def snapshot(): | |
| with _lock: | |
| return copy.deepcopy(_samples), dict(_status), dict(_alerted) | |
| def report(): | |
| """The route performs management GETs only; it never loads storage or sends alerts.""" | |
| account = read_account() | |
| samples, status, alerted = snapshot() | |
| alarm, cause = evaluate_alarm(account) | |
| return {**account, "samples": samples, "sampler": status, "alarm": alarm, | |
| "alarm_cause": cause, "alerted": alerted} | |
| def _clean_samples(samples): | |
| clean = {} | |
| for row in samples if isinstance(samples, list) else []: | |
| try: | |
| if not _stamp(row.get("at")) or not _stamp(row.get("generated_at")): | |
| continue | |
| usage = _money(row.get("month_to_date_usage")) | |
| if usage < 0: | |
| continue | |
| clean[row["at"]] = {"at": row["at"], "generated_at": row["generated_at"], | |
| "month_to_date_usage": usage, "source": BALANCE_SOURCE} | |
| except (AttributeError, TypeError, ValueError, OverflowError): | |
| continue | |
| return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:] | |
| def _clean_alerted(alerted): | |
| if not isinstance(alerted, dict): | |
| raise ValueError() | |
| for month, at in alerted.items(): | |
| if not isinstance(month, str) or not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", month) or not _stamp(at): | |
| raise ValueError() | |
| return {key: alerted[key] for key in sorted(alerted)[-2:]} | |
| def _sample_once(load, persist, *, sender=None): | |
| # Serialize manual and scheduled ticks as well as delivery marker changes. | |
| with _sample_lock: | |
| _sample_tick(load, persist, sender=sender) | |
| def _sample_tick(load, persist, *, sender=None): | |
| global _loaded, _samples, _alerted, _cluster_id | |
| account = read_account() | |
| balance = account["balance"] | |
| alarm, alarm_cause = evaluate_alarm(account) | |
| if not balance["available"]: | |
| with _lock: | |
| _status["cause"] = alarm_cause | |
| return | |
| selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID) | |
| with _lock: | |
| if _cluster_id != selected: | |
| _cluster_id, _loaded, _samples, _alerted = selected, False, [], {} | |
| _status["persisted"] = None | |
| _samples = _clean_samples([*_samples, {"at": balance["read_at"], | |
| "generated_at": balance["generated_at"], "month_to_date_usage": balance["month_to_date_usage"]}]) | |
| _status["last_at"] = balance["read_at"] | |
| awake, cause = store_is_awake() | |
| if not awake: | |
| with _lock: | |
| _status["cause"] = cause | |
| return | |
| try: | |
| if not _loaded: | |
| saved = load() | |
| saved = {} if saved is None else saved | |
| matching = saved.get("cluster_id") == selected | |
| history = _clean_samples(saved.get("samples", [])) if matching else [] | |
| markers = _clean_alerted(saved.get("alerted", {})) if matching else {} | |
| with _lock: | |
| _samples = _clean_samples([*history, *_samples]) | |
| _alerted, _loaded = markers, True | |
| except Exception: | |
| with _lock: | |
| _status["cause"] = "Sample history and alert markers could not be loaded; alerts deferred." | |
| return | |
| sent = False | |
| try: | |
| def save(): | |
| awake, cause = store_is_awake() | |
| if not awake: | |
| raise UsageReadError(cause) | |
| samples, _, markers = snapshot() | |
| persist({"cluster_id": selected, "samples": samples, "alerted": markers}) | |
| with _lock: | |
| _status.update(persisted=_now(), cause=alarm_cause) | |
| # Flush any successful delivery marker before considering another send. | |
| save() | |
| if not alarm or alarm["period"] in _alerted: | |
| return | |
| try: | |
| ok, delivery = (sender or send_telegram)(alarm["message"]) | |
| except Exception: | |
| ok, delivery = False, "Telegram alert could not be sent." | |
| if not ok: | |
| with _lock: | |
| _status["cause"] = delivery | |
| return | |
| sent = True | |
| with _lock: | |
| _alerted = _clean_alerted({**_alerted, alarm["period"]: _now()}) | |
| save() | |
| except Exception: | |
| with _lock: | |
| _status["cause"] = ( | |
| "Telegram alert sent; its marker remains pending in memory because storage is unavailable." | |
| if sent else "Sample history could not be stored; alerts deferred until storage is available.") | |
| def start_sampler(*, load=None, persist=None, sender=None): | |
| """Off by default; at most one sampler per process, with a one-minute floor.""" | |
| global _thread | |
| try: | |
| every_s = int(os.environ.get("AIOS_DO_SAMPLE_SECONDS", "0")) | |
| except ValueError: | |
| every_s = 0 | |
| if every_s <= 0 or load is None or persist is None: | |
| with _lock: | |
| _status.update(every_s=0, cause="Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset, invalid or zero, or storage callbacks are unavailable.") | |
| return False | |
| every_s = max(60, every_s) | |
| with _lock: | |
| if _thread is not None and _thread.is_alive(): | |
| return True | |
| _status.update(every_s=every_s, cause="Waiting for the first scheduled sample.") | |
| _stop.clear() | |
| def run(): | |
| while not _stop.is_set(): | |
| try: | |
| _sample_once(load, persist, sender=sender) | |
| except Exception: | |
| with _lock: | |
| _status["cause"] = "The scheduled sample could not be completed; it will retry." | |
| if _stop.wait(every_s): | |
| break | |
| _thread = threading.Thread(target=run, name="do-usage-sampler", daemon=True) | |
| _thread.start() | |
| return True | |
| def stop_sampler(): | |
| _stop.set() | |
| def _selftest(): | |
| """Run without credentials, sockets, runtimes, files or a real Telegram sender.""" | |
| import sys | |
| from unittest.mock import patch | |
| from core import neon_usage | |
| module = sys.modules[__name__] | |
| at = "2026-09-25T12:00:00Z" | |
| snapshot_at = "2026-09-25T00:00:00Z" | |
| database = {"id": DEFAULT_CLUSTER_ID, "name": "fixture-cluster", "size": "db-s-1vcpu-1gb", | |
| "num_nodes": 1, "region": "nyc3", "status": "online", "version": "18", | |
| "storage_size_mib": 10240, "connection": {"password": "fixture-secret"}} | |
| responses = { | |
| "/databases/" + DEFAULT_CLUSTER_ID: {"database": database}, | |
| "/customers/my/balance": {"month_to_date_usage": "20.00", "account_balance": "-1.00", | |
| "month_to_date_balance": "19.00", "generated_at": snapshot_at}, | |
| "/customers/my/billing_history": {"billing_history": []}, | |
| } | |
| calls, sends, documents = [], [], [] | |
| def fake_get(path, token): | |
| calls.append(path) | |
| return copy.deepcopy(responses[path]) | |
| def fake_sender(message): | |
| sends.append(message) | |
| return True, len(sends) | |
| def persist(document): | |
| documents.append(copy.deepcopy(document)) | |
| def reset(): | |
| global _loaded, _samples, _alerted, _cluster_id | |
| _loaded, _samples, _alerted, _cluster_id = False, [], {}, None | |
| sends.clear() | |
| documents.clear() | |
| # Double protection: all requests are fake, including if a test forgets sender=. | |
| with patch.dict(os.environ, {}, clear=True), patch.object(module, "_get", side_effect=fake_get), \ | |
| patch.object(module, "send_telegram", side_effect=fake_sender), \ | |
| patch.object(neon_usage, "send_telegram", side_effect=fake_sender), \ | |
| patch.object(module, "_now", return_value=at): | |
| missing = read_account() | |
| assert not missing["available"] and missing["cause"] and not calls | |
| assert missing["balance"]["month_to_date_usage"] is None | |
| assert missing["fixed_price"]["monthly_usd"] is None | |
| assert not start_sampler(load=lambda: {}, persist=persist, sender=fake_sender) | |
| assert "disabled" in report()["sampler"]["cause"] | |
| os.environ["DIGITALOCEAN_TOKEN"] = "fixture-fallback" | |
| account = read_account() | |
| assert account["fixed_price"]["monthly_usd"] == 15.15 | |
| assert account["balance"]["account_balance"] == -1 | |
| assert account["billing"]["available"] and account["billing_history"] == [] | |
| # The real empty history omits the key: {"links": {}, "meta": {}} is empty, not unreadable. | |
| responses["/customers/my/billing_history"] = {"links": {}, "meta": {}} | |
| enveloped = read_account() | |
| assert enveloped["billing"]["available"] and enveloped["billing_history"] == [] | |
| responses["/customers/my/billing_history"] = {"links": {}} | |
| assert not read_account()["billing"]["available"] | |
| responses["/customers/my/billing_history"] = {"billing_history": []} | |
| assert "fixture-secret" not in json.dumps(account) | |
| assert account["balance"]["source"] == BALANCE_SOURCE and account["balance"]["generated_at"] == snapshot_at | |
| os.environ["AIOS_DO_USAGE_TOKEN"] = "fixture-preferred" | |
| def preferred_get(path, token): | |
| assert token == "fixture-preferred" | |
| return fake_get(path, token) | |
| with patch.object(module, "_get", side_effect=preferred_get): | |
| assert read_account()["available"] | |
| for failed_path, failed_block in (("/databases/" + DEFAULT_CLUSTER_ID, "cluster"), | |
| ("/customers/my/balance", "balance"), | |
| ("/customers/my/billing_history", "billing")): | |
| def fail_one(path, token): | |
| if path == failed_path: | |
| raise UsageReadError("DigitalOcean API returned HTTP 403.") | |
| return fake_get(path, token) | |
| with patch.object(module, "_get", side_effect=fail_one): | |
| partial = read_account() | |
| assert partial["available"] and not partial[failed_block]["available"] | |
| assert "403" in partial[failed_block]["cause"] | |
| assert all(partial[key]["available"] for key in ("cluster", "balance", "billing") if key != failed_block) | |
| if failed_block == "balance": | |
| assert partial["balance"]["month_to_date_usage"] is None | |
| assert evaluate_alarm(partial)[0] is None and evaluate_alarm(partial)[1] | |
| _sample_once(lambda: {}, persist, sender=fake_sender) | |
| assert not sends and "403" in snapshot()[1]["cause"] | |
| original_balance = copy.deepcopy(responses["/customers/my/balance"]) | |
| for invalid in (None, True, "NaN", "Infinity", "unreadable"): | |
| responses["/customers/my/balance"]["month_to_date_usage"] = invalid | |
| malformed = read_account() | |
| assert not malformed["balance"]["available"] and malformed["balance"]["cause"] | |
| assert malformed["balance"]["month_to_date_usage"] is None | |
| assert evaluate_alarm(malformed)[0] is None | |
| responses["/customers/my/balance"] = original_balance | |
| database["size"] = "unknown-size" | |
| unknown = read_account() | |
| assert unknown["fixed_price"]["monthly_usd"] is None | |
| assert "UNKNOWN" in unknown["fixed_price"]["cause"] and "unknown-size" in unknown["fixed_price"]["cause"] | |
| assert evaluate_alarm(unknown)[0] is None | |
| database["size"] = "db-s-1vcpu-1gb" | |
| database["num_nodes"] = 2 | |
| assert read_account()["fixed_price"]["monthly_usd"] == 30.30 | |
| database["num_nodes"] = 1 | |
| reset() | |
| os.environ["AIOS_DO_ALARM_USD"] = "1" | |
| _sample_once(lambda: {}, persist, sender=fake_sender) | |
| _sample_once(lambda: documents[-1], persist, sender=fake_sender) | |
| assert len(sends) == 1 and sends[0].startswith("TEST ") | |
| assert all(word in sends[0] for word in ("20.00", "15.15", snapshot_at, "fixture-cluster", "R28")) | |
| saved = copy.deepcopy(documents[-1]) | |
| reset() | |
| _sample_once(lambda: saved, persist, sender=fake_sender) | |
| assert not sends and "2026-09" in documents[-1]["alerted"] | |
| responses["/customers/my/balance"]["generated_at"] = "2026-10-01T00:00:00Z" | |
| _sample_once(lambda: saved, persist, sender=fake_sender) | |
| assert len(sends) == 1 # A new billing month permits one new delivery. | |
| responses["/customers/my/balance"]["generated_at"] = snapshot_at | |
| reset() | |
| del os.environ["AIOS_DO_ALARM_USD"] | |
| responses["/customers/my/balance"]["month_to_date_usage"] = "15.15" | |
| _sample_once(lambda: {}, persist, sender=fake_sender) | |
| assert not sends | |
| responses["/customers/my/balance"]["month_to_date_usage"] = "0.00" | |
| zero = report() | |
| assert zero["balance"]["month_to_date_usage"] == 0 and not zero["alarm"] | |
| _sample_once(lambda: {}, persist, sender=fake_sender) | |
| assert not sends | |
| os.environ["AIOS_DO_ALARM_USD"] = "-1" | |
| _sample_once(lambda: {}, persist, sender=fake_sender) | |
| assert len(sends) == 1 and sends[0].startswith("TEST ") | |
| os.environ["AIOS_DO_ALARM_USD"] = "NaN" | |
| assert report()["alarm"] is None and "invalid" in report()["alarm_cause"] | |
| reset() | |
| os.environ["AIOS_DO_ALARM_USD"] = "-1" | |
| os.environ["STORE_BACKEND"] = "pg" | |
| os.environ["DATABASE_URL"] = "postgresql://fixture@ep-fixture.neon.tech/app" | |
| with patch.object(neon_usage, "read_project", return_value={"available": True, "id": "fixture"}), \ | |
| patch.object(neon_usage, "_get", return_value={"endpoints": [ | |
| {"id": "ep-fixture", "current_state": "idle"}]}): | |
| def forbidden(*args): | |
| raise AssertionError("Idle compute must never call storage") | |
| _sample_once(forbidden, forbidden, sender=fake_sender) | |
| assert not sends and "idle" in snapshot()[1]["cause"] | |
| assert report()["balance"]["available"] | |
| del os.environ["STORE_BACKEND"] | |
| del os.environ["DATABASE_URL"] | |
| with patch.object(neon_usage, "read_project", side_effect=AssertionError("Unexpected Neon read")): | |
| assert store_is_awake(backend="pg", database_url="postgresql://fixture@fixture.db.ondigitalocean.com/app")[0] | |
| assert not store_is_awake(backend="pg", database_url="")[0] | |
| assert not store_is_awake({"available": False, "cause": "offline"}, backend="pg", | |
| database_url="postgresql://fixture@ep-fixture.neon.tech/app")[0] | |
| reset() | |
| def failed_load(): | |
| raise RuntimeError("fixture-secret") | |
| _sample_once(failed_load, persist, sender=fake_sender) | |
| assert not sends and "loaded" in snapshot()[1]["cause"] | |
| reset() | |
| def failed_persist(document): | |
| raise RuntimeError("fixture-secret") | |
| _sample_once(lambda: {}, failed_persist, sender=fake_sender) | |
| assert not sends and "stored" in snapshot()[1]["cause"] | |
| assert "fixture-secret" not in json.dumps(report()) | |
| reset() | |
| def fail_after_delivery(document): | |
| if sends: | |
| raise RuntimeError("fixture-secret") | |
| persist(document) | |
| _sample_once(lambda: {}, fail_after_delivery, sender=fake_sender) | |
| assert len(sends) == 1 and "pending in memory" in snapshot()[1]["cause"] | |
| _sample_once(lambda: {}, persist, sender=fake_sender) | |
| assert len(sends) == 1 and "2026-09" in documents[-1]["alerted"] | |
| rows = [{"at": f"2026-09-{1 + n // 24:02}T{n % 24:02}:00:00Z", "generated_at": snapshot_at, | |
| "month_to_date_usage": n} for n in range(MAX_SAMPLES + 1)] | |
| assert len(_clean_samples(rows)) == MAX_SAMPLES | |
| # Exercise the actual HTTP boundary with a fake opener, including credential-bearing errors. | |
| class Response: | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, *args): | |
| pass | |
| def read(self, limit): | |
| return b'{"ok": true}' | |
| class Opener: | |
| def open(self, request, *, timeout): | |
| assert request.get_method() == "GET" and timeout == 5 | |
| return Response() | |
| with patch.object(module, "build_opener", return_value=Opener()) as opener: | |
| assert _get("/fixture", "fixture-token") == {"ok": True} | |
| assert isinstance(opener.call_args.args[0], _NoRedirect) | |
| assert opener.call_args.args[0].redirect_request(None, None, 302, None, None, None) is None | |
| for error in (HTTPError("fixture-private-url", 403, "fixture-token", None, None), | |
| URLError("fixture-private-url"), TimeoutError("fixture-token"), ValueError("fixture-token")): | |
| with patch.object(module, "build_opener", side_effect=error): | |
| try: | |
| _get("/fixture", "fixture-token") | |
| raise AssertionError("The HTTP error must fail closed") | |
| except UsageReadError as exc: | |
| assert "fixture-" not in str(exc) | |
| print("PASS DigitalOcean usage selftest: isolated API failures, pricing, snapshot provenance, monthly alerts, restart markers, idle-store guard and safe HTTP errors; fake senders only.") | |
| if __name__ == "__main__": | |
| import sys | |
| if sys.argv[1:] != ["--selftest"]: | |
| raise SystemExit("Usage: python -m core.do_usage --selftest") | |
| _selftest() | |