from __future__ import annotations import os import asyncio import gc import sqlite3 import sys import time import uuid from collections.abc import Callable from contextlib import asynccontextmanager, suppress from pathlib import Path from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response from fastapi.exceptions import RequestValidationError from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from starlette.exceptions import HTTPException as StarletteHTTPException from sinetti_api.distribution import Distribution, distribution, load_full_registration if distribution() is Distribution.FULL: import secrets_local from sinetti_services import serving_recovery import obs import state_paths from sinetti_api.addons import batch_check, market_balances from sinetti_api.upload_limits import BatchCheckUploadLimitMiddleware if distribution() is Distribution.FULL: # Mirror Classic (app.py): load AI/search keys from a local .env for APP_BACKEND=local runs. # Gated + no-ops on HF (existing env / Space Secrets win). Without this, V2 uvicorn starts # with a bare environment and Sentinel/chat see a missing ANTHROPIC_API_KEY even though .env # has it. The fixed CUSTOMER composition does not ship the desktop dotenv helper. secrets_local.load_local_dotenv() # Private mode (Private Cloud): arm the outbound-connection guard before anything - boot data # sync, prewarm, any router - can open a socket. No-op unless SINETTI_PRIVATE_MODE=1. from sinetti_services import egress_guard as _egress_guard _egress_guard.install() from sinetti_services import diagnostics_contract as _diagnostics # noqa: E402 from sinetti_api.deps import ( require_admin, require_data_access, require_feature, require_research_data, research_scope, require_user, ) from sinetti_api.errors import ( ActionUnavailable, FeatureBlocked, action_unavailable_handler, feature_blocked_handler, http_exception_handler, request_validation_exception_handler, tenant_required_handler, unhandled_exception_handler, ) from sinetti_services.tenant_context import TenantRequired from sinetti_api.routers import ( about, access_public, ask_conversations, discussions, analytics, archive_life, auth, business_footprint, capabilities, category_timelines, certificates, certification_bodies, chat, contact, feedback, counterparty, dossier, email_public, feedstocks, flags, fuel_conversion, fuel_specs, intro_video, fuels, geography, health, holders, maps, material_families, me_profile, monitor, narrative, notifications, ops, places, product_signals, risk, saf_exports, activity, saved_views, search, security, sentinel, site_footprint, site_assessment, soft_presets, timeline, tours, watchlist, ) # Kept in sync with sinetti_api/routers/feedstocks.py's _PATHWAY_FAMILIES so a future family # added to the router is automatically prewarmed without touching this module. _PREWARM_FAMILIES = tuple(sorted(feedstocks._PATHWAY_FAMILIES)) # Once the shared lock budget below is spent, every remaining target waits only this long -- # enough to take an uncontended lock, not enough to add another wait per component. Measured # 2026-08-13: with a 250 ms floor instead, one contended readiness poll spent ~250-800 ms on # EACH of the seven targets in turn and /api/ready peaked at 3,584 ms, against a 1 s budget # that was only ever enforced on the first target. PERSISTENCE_PROBE_MIN_TIMEOUT_MS = 25 # A locked SQLite file is not the same fault as a missing table. The lock is held by this # app's own in-flight write and clears in milliseconds; the platform's remedy for an # unready container -- restart it, or take it out of rotation -- clears nothing and moves # that traffic onto the sibling worker, which shares the same file. So the probe waits for # a lock it can still get instead of reporting the app unhealthy for serving customers. # The budget is shared across every target so /api/ready stays inside the poll timeouts # that scripts/preflight_deploy.py and scripts/assert_v2_serve_matches_desk.ps1 already use. PERSISTENCE_PROBE_LOCK_BUDGET_MS = 1000 # Waiting alone would only move the brittleness: a wedged writer would still flip the # container unhealthy, just later. A lock that survives this many consecutive readiness # probes has stopped being contention and IS a not-ready condition, so the signal survives. PERSISTENCE_PROBE_CONTENTION_TOLERANCE = 1 _INTEL_REQUIRED_SCHEMA = { "iscc_certs": frozenset({ "certificate_id", "holder_name", "holder_norm", "country", "scope", "proc_unit_type", "feedstocks", "add_ons", "products", "valid_from", "valid_until", "suspended_date", "issuing_cb", "raw_status", }), "iscc_sync_log": frozenset({ "id", "synced_at", "n_total", "n_new", "n_withdrawn", "n_renewed", "deltas_json", "source", }), "iscc_compliance_history": frozenset({"id", "snapshot_at", "source", "kind", "members_json"}), "iscc_tracked": frozenset({ "org_id", "user_id", "holder_norm", "label", "pinned_at", "list_name", "alerts", }), "iscc_saved_views": frozenset({"org_id", "user_id", "scheme", "name", "payload", "created_at"}), "iscc_saved_view_shares": frozenset({ "share_id", "created_by_user_id", "created_at", "expires_at", "suggested_name", "scheme", "payload", }), "iscc_feedstock_alias": frozenset({"feedstock", "alias"}), "iscc_timeline_sets": frozenset({"user_id", "name", "scheme", "holder_norms", "created_at"}), "holder_bd_research": frozenset({ "org_id", "holder_norm", "scheme", "holder_name", "country", "researched_at", "search_provider", "search_query_count", "ai_provider", "report_json", "evidence_json", "status", "error", }), "producer_cert_links": frozenset({"producer_id", "certificate_id"}), "iscc_alerts": frozenset({"id", "holder_norm", "dedup_key", "kind", "title", "detail", "created_at"}), "dismissals": frozenset({"org_id", "user_id", "dedup_key", "dismissed_at"}), "notification_preferences": frozenset({ "org_id", "user_id", "delivery_mode", "destination", "destination_verified", "unsubscribed", "updated_at", }), "user_profile": frozenset({"org_id", "user_id", "work_role", "updated_at"}), "notification_delivery_attempts": frozenset({ "idempotency_key", "org_id", "user_id", "destination", "event_ids_json", "final_status", "attempt_count", "claim_generation", "created_at", "updated_at", "attempted_at", "next_attempt_at", "provider_response_id", }), "notification_event_claims": frozenset({ "org_id", "user_id", "destination_key", "event_id", "attempt_idempotency_key", "claimed_at", "updated_at", }), "web_push_subscriptions": frozenset({ "subscription_id", "org_id", "user_id", "endpoint", "p256dh", "auth", "created_at", "updated_at", "disabled_at", }), "web_push_delivery_claims": frozenset({ "subscription_id", "event_id", "org_id", "user_id", "state", "claimed_at", "updated_at", "provider_status", }), "web_push_delivery_queue": frozenset({ "subscription_id", "event_id", "org_id", "user_id", "holder_norm", "scheme", "enqueued_at", "next_attempt_at", }), "login_lockout": frozenset({"username", "fail_count", "locked_until"}), } _INTEL_REQUIRED_INDEXES = frozenset({ "ix_notification_attempts_user", "ix_notification_attempts_retry", "ix_notification_event_claims_attempt", "ix_web_push_subscriptions_tenant", "ix_web_push_delivery_claims_tenant", "ix_web_push_delivery_queue_oldest", "ix_web_push_delivery_queue_tenant", "ix_web_push_delivery_queue_ready", "ix_iscc_holder", "ix_iscc_country", }) _ENRICH_REQUIRED_SCHEMA = { "enrichment_cache": frozenset({ "provider", "input_key", "status", "payload_json", "fetched_at", "expires_at", }), "enrichment_history": frozenset({ "id", "org_id", "holder_norm", "scheme", "verdict_level", "reasons_json", "produced_at", "produced_by", }), } _AI_REQUIRED_SCHEMA = { "ai_usage": frozenset({ "id", "user_id", "feature", "model", "input_tokens", "output_tokens", "cache_write_tokens", "cache_read_tokens", "search_queries", "cost_usd", "created_at", }), "ai_limits": frozenset({"user_id", "feature", "monthly_usd", "updated_at", "updated_by"}), "ai_reservations": frozenset({ "reservation_id", "user_id", "feature", "reserved_usd", "created_at", "expires_at", }), } _AI_REQUIRED_INDEXES = frozenset({"idx_ai_usage_user_month", "idx_ai_reservations_user_feature"}) _DATA_ACCESS_REQUIRED_SCHEMA = { "data_access_limits": frozenset({"user_id", "requests_per_window", "daily_export_rows", "monthly_export_rows", "periodic_unlimited", "updated_at", "updated_by"}), "data_request_events": frozenset({"event_id", "user_id", "occurred_at"}), "data_access_org_limits": frozenset({"org_id", "requests_per_window", "daily_export_rows", "monthly_export_rows", "periodic_unlimited", "updated_at", "updated_by"}), "data_org_request_events": frozenset({"event_id", "org_id", "occurred_at"}), "data_export_audit": frozenset({"audit_id", "trace_id", "created_at", "user_id", "org_id", "is_admin", "method", "route_name", "schemes_json", "filters_json", "exact_selection_count", "exact_selection_sha256", "row_count", "byte_count", "proposed_filename", "outcome", "reason"}), "data_limit_changes": frozenset({ "change_id", "target_kind", "target_id", "changed_at", "changed_by", "old_settings_json", "new_settings_json", }), } _DATA_ACCESS_REQUIRED_INDEXES = frozenset({ "data_request_events_user_time_idx", "data_org_request_events_org_time_idx", "data_export_audit_time_idx", "data_export_audit_user_time_idx", "data_export_audit_org_time_idx", "data_limit_changes_target_idx", }) _AUTH_REQUIRED_SCHEMA = { # Exactly what session validation reads: a drifted auth store silently turns every # issued token into a guest (validate() swallows store errors), which is invisible to # every gate until the live smoke fails — probe it like the rest. "auth_sessions": frozenset({ "session_digest", "user_id", "auth_version", "issued_at", "last_seen_at", "expires_at", "device_label", "revoked_at", "revoked_by", "revoke_reason", }), "auth_account_controls": frozenset({ "user_id", "state", "auth_version_delta", "changed_at", "changed_by", "reason", }), } _AUTH_REQUIRED_INDEXES = frozenset({"auth_sessions_user_active_idx"}) _OPS_ERROR_REQUIRED_SCHEMA = { "ops_error_events": frozenset({ "event_id", "created_at", "source", "trace_id", "user_id", "path", "component", "message", "fingerprint", }), } _OPS_ERROR_REQUIRED_INDEXES = frozenset({ "ops_error_events_created_idx", "ops_error_events_source_created_idx", "ops_error_events_fingerprint_idx", }) _OPS_OWNERSHIP_REQUIRED_SCHEMA = { "ops_ownership_events": frozenset({ "event_id", "created_at", "actor_user_id", "org_id", "action", "detail", }), } _OPS_OWNERSHIP_REQUIRED_INDEXES = frozenset({ "ops_ownership_events_created_idx", "ops_ownership_events_org_created_idx", }) _BILLING_REQUIRED_SCHEMA = { "billing_orgs": frozenset({ "org_id", "name", "status", "seats", "stripe_customer_id", "created_at", "updated_at", }), "billing_org_members": frozenset({"org_id", "user_id", "added_at"}), "billing_plans": frozenset({ "plan_id", "name", "features", "schemes", "seat_limit", "ai_budgets", "created_at", "updated_at", }), "billing_plan_assignments": frozenset({ "org_id", "plan_id", "current_period_start", "current_period_end", "source", "updated_at", "updated_by", }), } def _prewarm_critical() -> Exception | None: """Critical readiness warm: corpus + Browse + pathway index + landing feedstock cards. This is the path the Browse landing page and feedstock-map capacity probe hit first. Gzipped World-list warm alone does not cover POST /api/certificates/query. Pathways + the certificate-pathway index are warmed here so card builds do not pay a cold index, and the first page of default World Browse holders get their feedstock-card responses built so the first real fan-out is a cache hit. Best-effort: failures fall through to the normal on-request load path. """ try: import iscc import sinetti_data from sinetti_api.routers import certificates as _certs from sinetti_api.routers import counterparty as _counterparty from sinetti_api.routers import holders as _holders from sinetti_services import pathways_service from sinetti_services.holder_name_index import holder_name_index_for from sinetti_services.read_models import list_certificate_summaries groups = iscc._all_scheme_certs() _counterparty.prewarm_fit_index(groups) list_certificate_summaries(groups, scheme="World", status="all") # Browse/Verify typeahead: prefix index over raw rows (avoids cold first keystroke). holder_name_index_for(groups, scheme="World") _certs.prewarm_default_browse_query(groups) sinetti_data.load_app_pathways() from sinetti_data_reads.curated_results import customer_mode if not customer_mode(): # Customer pathway indexes live on each bound result snapshot, built on first read. pathways_service._certificate_pathway_index() _holders.prewarm_default_browse_feedstock_cards(groups) # Every page loads /api/material-families; cold it built the payload on the first request. from sinetti_services import family_resolution family_resolution.build_families_payload_bytes() audit_failure = _prewarm_audit_details() if audit_failure is not None: obs.warn("prewarm_audit_details_failed", error=type(audit_failure).__name__) except Exception as exc: return exc return None def _prewarm_audit_details() -> Exception | None: """Unpack the certificate-PDF details file before the first card asks for an audit. Each Browse card fetches its audits; cold, the first ones unpacked the 1.2 GB details copy and the whole server waited (SIN-0996). The customer edition reads released results instead and its provider returns None here. """ try: from sinetti_services import documents_source documents_source.audit_details_path() except Exception as exc: return exc return None def _prewarm_search() -> Exception | None: """Build the name-search indexes so the first text search after boot is fast. Verify (full search) and Browse each match through a token index built once per corpus (sinetti_domain.name_search.NameSearchIndex, a few seconds cold). The default Browse probe can resolve to suggestions before the filter needs its index, so explicitly build the canonical World summary index after that probe. Called after selected startup phases so later warm work cannot evict these bounded indexes before readiness. """ try: import iscc from sinetti_domain.metrics import deduplicate_registry_rows from sinetti_domain.name_search import name_search_index_for from sinetti_services.search_service import search_omnibox from sinetti_services.read_models import list_certificate_summaries from sinetti_api.routers import certificates as _certs groups = iscc._all_scheme_certs() search_omnibox(groups, "prewarm", scheme="World") # Keep the ordinary Browse query warm, then materialize the exact index its canonical # World/all filter uses, including when the dummy holder probe resolves to suggestions. _certs.prewarm_default_browse_query(groups, holder="prewarm") canonical_rows = deduplicate_registry_rows( list_certificate_summaries(groups, scheme="World", status="all") ) name_search_index_for(canonical_rows, aliases_field="entity_aliases") except Exception as exc: return exc return None def _prewarm_optional() -> Exception | None: """Best-effort deep-dive / sites / list-bytes / CB readiness warm. Skipped only when SINETTI_PREWARM_OPTIONAL=0 (or SINETTI_PREWARM=0). Must build under the EXACT memo key real requests use (sinetti_api/routers/feedstocks.py), or this warms a key no request ever reads. That means: the same expiry_days/delisted_ids as get_feedstock_context() (build_family_deepdive's memo key includes int(expiry_days), and defaults to 0 if omitted -- a permanent miss against the real default of 90), and passing `groups` into build_fwo_deepdive (by_scheme=None short-circuits its memo write entirely). """ try: import iscc import sinetti_data import sinetti_services.deepdive_service as deepdive_service from sinetti_api.deps import get_feedstock_context from sinetti_services.feedstock_deepdive import build_feedstock_deepdive groups = iscc._all_scheme_certs() sinetti_data.load_app_pathways() sinetti_data.load_app_sites() context = get_feedstock_context() # Site markers first: Browse (Source near, card framing), Producers and Certified Sites # all wait on /api/maps/sites, and after the deep-dives below it stayed cold for the # first minute of traffic after a restart (11-26 s cold, 2026-09-30). _prewarm_site_markers(groups, context) _prewarm_near_measurement(groups) for family in _PREWARM_FAMILIES: deepdive_service.build_family_deepdive( family, groups, scheme="World", status="active", expiry_days=context["expiry_days"], delisted_ids=context["delisted_ids"], ) # supply_chain is attached on every family response and has its own memo — # warm the default active-status key the Feedstocks page requests. build_feedstock_deepdive( family, groups, filters={"statuses": ["active"]}, expiry_days=context["expiry_days"], ) deepdive_service.build_fwo_deepdive(groups, scheme="World", status="active") # Fuels: /api/fuels/{family}/deepdive serves biodiesel/ethanol/biogas through this SAME # build_family_deepdive memo, but _PREWARM_FAMILIES lists only feedstock families, so # those three were never warmed anywhere. Measured cost of a cold one on the 2026-08 # corpus: 60-67 s on an idle machine, which is what timed out deepdive-evolution in the # release gate and is what the first user after a Space boot would have paid. HVO is # already covered by _prewarm_landing_deepdive. The route passes empty regions/countries # sets and the memo key normalises `regions or ()`, so omitting them hits the same key. from sinetti_api.routers.fuels import _FUEL_FAMILIES for fuel in _FUEL_FAMILIES: if fuel == "hvo": continue deepdive_service.build_family_deepdive( fuel, groups, scheme="World", status="active", expiry_days=context["expiry_days"], delisted_ids=context["delisted_ids"], ) # #62: warm the memoized+gzipped World/all list bytes too, so the first Browse paint after # a cold start is served from cache instead of paying the full build+encode+compress cost. from sinetti_api.routers import certificates as _certs _certs._gzipped_list_bytes(groups, "World", "all") # CB deep-dive: warm certificate rows + facts cube + year series (gallery cold path ~20s). sinetti_data.load_app_certificate_rows() from sinetti_api.routers import certification_bodies as _cb _cb.prewarm_cb_deepdive() # Geography: warm the world matrix (~2 s cold) — it is the deep dive's default view. from sinetti_api.routers import geography as _geo _geo.prewarm_geography_deepdive() _prewarm_map_overlays() _prewarm_holder_page() _prewarm_ask_library() except Exception as exc: return exc return None def _prewarm_site_markers(groups: dict[str, list[dict]], context: dict) -> None: """Build the site markers and the gzipped full-plan /api/maps/sites body under route keys.""" from datetime import date import sinetti_data from sinetti_services.map_service import list_site_markers site_markers = list_site_markers( sinetti_data.load_app_sites(), certs=groups, today=date.today().isoformat(), expiry_days=context["expiry_days"], delisted_ids=context["delisted_ids"], ) # The serialized body has its own guard: a bad marker must not cost the warms after it (the # request path fails open to an empty envelope on the same error). try: from sinetti_api.routers import maps as _maps _maps.warm_site_markers_payload(site_markers) except Exception: obs.warn("prewarm_site_markers_payload_failed") def _prewarm_browse_maps() -> Exception | None: """Warm what the first filtered Browse page builds besides its list (SIN-0995). A filtered Browse page fires the list query, two /api/maps/holder-coords/query calls and /api/maps/sites at once. Cold, those builds plus the delisted-id scan and the first pathway material scan shared one interpreter lock: the UCO + Europe list answered after 60-110 s against 2 s alone (2026-10-25). Runs before bind, so on the :8511 proxy the old backend keeps serving until this one is warm. """ try: import iscc from sinetti_api.deps import get_feedstock_context from sinetti_api.routers import certificates as _certs from sinetti_api.routers import maps as _maps from sinetti_services import map_service groups = iscc._all_scheme_certs() context = get_feedstock_context() _prewarm_site_markers(groups, context) map_service.warm_holder_coords(groups) map_service.warm_browse_country_centres() _maps._sibling_norms(groups) # One real material-filtered query: entitled pathway rows, unit evidence and the # material label memo every feedstock filter reads. _certs.prewarm_default_browse_query(groups, feedstocks=["Used cooking oil (UCO)"]) except Exception as exc: return exc return None def _prewarm_near_measurement(groups: dict[str, list[dict]]) -> None: """Build the indexes behind every "near" measurement (Browse Source near, the map's ring). The postal locality index, the physical-site points and the per-scheme register points took ~9 s on the first Source query after a restart (2026-10-11), paid inside that request. They memoise per scheme name, never per caller, so one measurement over the whole corpus through the real path warms the keys every entitled subset reads. A request abandoned mid-build keeps building them on purpose (SIN-0150), so warming here is what takes them off the request path. """ from sinetti_services import browse_near from sinetti_services.read_models import list_certificate_summaries browse_near.holder_distances( list_certificate_summaries(groups, scheme="World", status="all"), lat=0.0, lon=0.0, radius_km=browse_near.normalize_radius_km(None), allowed_schemes=groups, ) def _prewarm_map_overlays() -> None: """Warm the Browse map overlays the first page load fetches alongside the certificate list. The hosted run log of 2026-09-29 showed these as the slowest first calls after a restart: JSON SOURCE producers 19.6 s (28 s cold on the desk: it builds the scheme="All" producer markers it suppresses against) and ISCC life bounds 22 s (3.5 s alone). Both memoise per process, so building them here makes the first customer request a cache hit. Called with no arguments, exactly like their routes, so the warm lands on the key requests read. """ from sinetti_services import sites_source from sinetti_services.iscc_life_bounds import get_iscc_life_bounds get_iscc_life_bounds() sites_source.json_source_markers() sites_source.research_gap_markers() if sites_source.mode() == sites_source.MODE_RELEASED: # The seam already logged one ERROR line if the file is unusable; this is the one # INFO line an operator reads to know which release scope is being served. obs.info("released_sites_status", **sites_source.status()) def _prewarm_holder_page() -> None: """Warm the per-process caches every holder dossier reads, whichever holder it is. Cold profile of 2026-09-30 (Neste dossier right after a restart, 2.3 s against 0.4 s warm): the site-country locator behind register sites took ~1.1 s to build over every site row, and the related-company registry files ~0.3 s to read. Both memoise per process. """ from sinetti_services import site_country from sinetti_data_reads.curated_results import customer_mode as _customer_results_mode if _customer_results_mode(): from sinetti_data_reads import company_results as company_external_relations else: from sinetti_services import company_external_relations site_country.default_locate() company_external_relations.load_registry_decisions() def _prewarm_ask_library() -> None: """Build Ask's per-entitlement retrieval views (SIN-0572) for the plans in use. Every authenticated question filters the library by the asker's schemes; a cold view costs ~1.3 s on the first question per plan. Local only: no question is embedded. """ import rag_store from sinetti_services import entitlements rag_store.prewarm(entitlements.ask_scheme_sets()) def _prewarm_uco_deepdive() -> Exception | None: """Warm only the default Feedstocks (UCO) family and supply-chain payloads.""" try: import iscc import sinetti_services.deepdive_service as deepdive_service from sinetti_api.deps import get_feedstock_context from sinetti_services.feedstock_deepdive import build_feedstock_deepdive groups = iscc._all_scheme_certs() context = get_feedstock_context() deepdive_service.build_family_deepdive( "uco", groups, scheme="World", status="active", expiry_days=context["expiry_days"], delisted_ids=context["delisted_ids"], ) build_feedstock_deepdive( "uco", groups, filters={"statuses": ["active", "expiring"]}, expiry_days=context["expiry_days"], ) except Exception as exc: return exc return None def _prewarm_landing_deepdive() -> Exception | None: """Warm default Feedstocks (UCO) + Biofuels (HVO) landings before readiness. The first open of Feedstocks always lands on UCO; without this warm the user pays a cold ~3 s family build + ~1 s supply_chain build. Biofuels lands on HVO; family memo alone is not enough — build_fuels_deepdive still pays pathway rescans + crosstab (~2 s) unless the full fuels payload is warmed. The full multi-family warm defaults on and can be skipped with SINETTI_PREWARM_OPTIONAL=0. """ try: failure = _prewarm_uco_deepdive() if isinstance(failure, Exception): return failure import iscc import sinetti_services.fuels_deepdive as fuels_deepdive from sinetti_api.deps import get_feedstock_context groups = iscc._all_scheme_certs() context = get_feedstock_context() fuels_deepdive.build_fuels_deepdive( groups, scheme="World", status="active", expiry_days=context["expiry_days"], delisted_ids=context["delisted_ids"], ) except Exception as exc: return exc return None def _prewarm_data() -> None: """Full warm (critical Browse/summary + optional deep-dives). Used by e2e sync path.""" _run_prewarm_phase("critical", _prewarm_critical) _run_prewarm_phase("landing", _prewarm_landing_deepdive) _run_prewarm_phase("browse", _prewarm_browse_maps) _run_prewarm_phase("optional", _prewarm_optional) _run_prewarm_phase("search", _prewarm_search) def _run_prewarm_phase( phase: str, target: Callable[[], Exception | None] ) -> None: """Run one readiness warm best-effort and expose its startup cost.""" started = time.perf_counter() obs.info("prewarm_phase_begin", phase=phase) try: failure = target() except Exception as exc: # noqa: BLE001 — warm misses fall back to request-time builds failure = exc duration_s = round(time.perf_counter() - started, 3) if isinstance(failure, Exception): obs.error( "prewarm_phase_failed", phase=phase, duration_s=duration_s, error_type=type(failure).__name__, error=str(failure), ) else: obs.info( "prewarm_phase_done", phase=phase, duration_s=duration_s, ) def _prewarm_before_ready() -> None: """Synchronously warm selected deep dives before the application reports ready. The broad optional warm defaults on because its cold misses measured up to 60-80 seconds on the 2026-08 corpus. SINETTI_PREWARM_LANDING=0 skips the UCO/fuels landing warm; SINETTI_PREWARM_BROWSE=0 skips the first-Browse map warm; SINETTI_PREWARM_OPTIONAL=0 skips the remaining broad phase but keeps the final Verify/Browse search warm; SINETTI_PREWARM=0 disables warming here and the critical phase in the lifespan. Local serve scripts skip landing+optional so bind is not blocked for minutes; HF/prod keep the defaults. """ if os.environ.get("SINETTI_PREWARM", "1") == "0": obs.info("prewarm_before_ready_skipped", reason="SINETTI_PREWARM=0") return landing_disabled = os.environ.get("SINETTI_PREWARM_LANDING", "1") == "0" browse_enabled = os.environ.get("SINETTI_PREWARM_BROWSE", "1") != "0" if landing_disabled: obs.info("prewarm_phase_skipped", phase="landing", reason="SINETTI_PREWARM_LANDING=0") else: _run_prewarm_phase("landing", _prewarm_landing_deepdive) if not browse_enabled: obs.info("prewarm_phase_skipped", phase="browse", reason="SINETTI_PREWARM_BROWSE=0") else: _run_prewarm_phase("browse", _prewarm_browse_maps) if os.environ.get("SINETTI_PREWARM_OPTIONAL", "1") == "0": obs.info("prewarm_phase_skipped", phase="optional", reason="SINETTI_PREWARM_OPTIONAL=0") else: _run_prewarm_phase("optional", _prewarm_optional) _run_prewarm_phase("search", _prewarm_search) # Collector settings applied once the boot heap is warm (SIN-0486). A full pass over the warmed # heap (~9.3 million tracked objects) held the interpreter lock for 4-5 s on the owner PC, longer # on the Space's shared cores, so uvicorn's 5 s worker health check killed the worker. Freezing # the boot heap removes it from every later pass; the raised gen-0 threshold makes young passes # rarer. Measured against a burst of 75 cold Browse searches. GC_THRESHOLDS = (50_000, 20, 1_000) # Any collector pass at least this long is logged with its generation and duration. GC_SLOW_PASS_SECONDS = 0.5 _gc_pass_started_at = 0.0 # Bound once: the collector fires inside any request, so the callback must not share the # module-level ``time`` that request timing (and its tests) read. _gc_clock = time.perf_counter def _log_slow_gc_pass(phase: str, info: dict) -> None: """gc.callbacks hook: time every pass, log only the slow ones (diagnostics, never raises).""" global _gc_pass_started_at if phase == "start": _gc_pass_started_at = _gc_clock() return duration = _gc_clock() - _gc_pass_started_at if duration < GC_SLOW_PASS_SECONDS: return try: obs.warn( "gc_pass_slow", pid=os.getpid(), generation=info.get("generation"), duration_ms=round(duration * 1000), collected=info.get("collected"), frozen_objects=gc.get_freeze_count(), ) except Exception: # noqa: BLE001 — a logging failure must not escape into the collector pass def _install_gc_pause_logger() -> None: if _log_slow_gc_pass not in gc.callbacks: gc.callbacks.append(_log_slow_gc_pass) def _freeze_boot_heap() -> None: """Collect once, freeze the warmed boot heap and raise the collector thresholds. SINETTI_GC_FREEZE=0 disables it. Skipped with SINETTI_PREWARM=0 too: without a warm there is no boot heap worth freezing, and the test suite would pay a full collection per app start. """ if os.environ.get("SINETTI_GC_FREEZE", "1") == "0": obs.info("gc_freeze_skipped", reason="SINETTI_GC_FREEZE=0") return if os.environ.get("SINETTI_PREWARM", "1") == "0": obs.info("gc_freeze_skipped", reason="SINETTI_PREWARM=0") return started = time.perf_counter() collected = gc.collect() collect_ms = round((time.perf_counter() - started) * 1000) gc.freeze() gc.set_threshold(*GC_THRESHOLDS) frozen = gc.get_freeze_count() obs.info( "gc_boot_heap_frozen", pid=os.getpid(), collect_ms=collect_ms, collected=collected, frozen_objects=frozen, tracked_objects=frozen + len(gc.get_objects()), thresholds=list(GC_THRESHOLDS), ) # Python's default 5 ms GIL switch interval starves I/O-bound threads behind CPU-bound ones: a # thread that releases the GIL for a SQLite row step or a stat() waits up to the whole interval to # get it back. A cold RSO deep-dive next to one CPU-busy request went 4.0 s -> 80.7 s, and three # heavy requests on a cold desk took 19-22 minutes each (2026-10-17). 0.5 ms measured 80.7 -> 23.2 s. GIL_SWITCH_INTERVAL_SECONDS = 0.0005 def _shorten_gil_switch_interval() -> None: """Set the process GIL switch interval. SINETTI_GIL_SWITCH_INTERVAL overrides it (seconds); 0 keeps the interpreter default.""" raw = os.environ.get("SINETTI_GIL_SWITCH_INTERVAL", "").strip() try: interval = float(raw) if raw else GIL_SWITCH_INTERVAL_SECONDS except ValueError: interval = GIL_SWITCH_INTERVAL_SECONDS if interval <= 0: obs.info("gil_switch_interval_kept", seconds=sys.getswitchinterval()) return sys.setswitchinterval(interval) obs.info("gil_switch_interval_set", seconds=sys.getswitchinterval()) @asynccontextmanager async def _lifespan(app: FastAPI): obs.enable_worker_faulthandler() _install_gc_pause_logger() _shorten_gil_switch_interval() recovery = serving_recovery.recover_pending_activation( data_dir=state_paths.BUNDLED_DATA_DIR ) obs.info("data_release_promotion_recovery", **recovery) if recovery.get("status") not in ("clean", "restored"): raise RuntimeError( str(recovery.get("error") or "serving release promotion recovery failed") ) import release_store heal = release_store.boot_self_heal() obs.info("release_store_boot", **heal) import sinetti_data sinetti_data.active_artifact_path.cache_clear() # resolve fresh after any self-heal switch from sinetti_data_reads.curated_results import customer_mode if customer_mode(): from sinetti_data_reads.material_results import install_feedstock_providers install_feedstock_providers() resolution = release_store.resolve_active() obs.info("serving_resolution", **{k: str(v) for k, v in resolution.items()}) if not Path(resolution["sqlite_path"]).is_file(): # Resolution succeeded (never raises) but the file it points at doesn't exist. The most # common cause: a fresh `git worktree add` never receives the gitignored serving data # pair (see release_store.PAIR_NAMES) — every read silently returns [] (sinetti_data. # _read_rows) instead of failing, so endpoints look "empty" rather than broken. This # never blocks boot: prod always has the bundled file, and pytest/CI intentionally # exercise the empty-corpus fallback. obs.error( "serving_artifact_missing", sqlite_path=str(resolution["sqlite_path"]), source=resolution["source"], reason=resolution["reason"], hint="corpus data file not found on disk -- if this is a fresh git worktree, run " "'python scripts/provision_worktree_data.py' to copy it from the main checkout", ) # Resolve the serving release before initializing state-backed persistence. The optional # seeders below depend on both boundaries: first_run_load reads the resolved serving corpus # and writes compliance/watchlist state, while saved views require the initialized intel DB. _initialize_required_persistence(app) _run_optional_startup_seeders() if distribution() is Distribution.FULL: load_full_registration().preflight_cert_format_catalog() try: from sinetti_data_reads.curated_results import customer_mode as _customer_results_mode from sinetti_services import lifecycle_source if not lifecycle_source.history_available(): if _customer_results_mode(): # Expected on every customer install (owner decision 2026-10-22, option b): # no refresh history ships yet, so this is information, not an operator fault. obs.info( "delisting_inference_unavailable", path=str(state_paths.refresh_dir() / "change_events.jsonl"), hint="customer install: no change history ships with this release, so " "Ask's lifecycle_history tool cannot detect delisted certificates -- " "every 'delisted: false' it reports means 'could not check', not " "'confirmed clean'. Change history arrives with a later data release; " "nothing to run on this host.", ) else: obs.error( "delisting_inference_unavailable", path=str(state_paths.refresh_dir() / "change_events.jsonl"), hint="Ask's lifecycle_history tool cannot detect delisted certificates on " "this install -- every 'delisted: false' it reports means 'could not " "check', not 'confirmed clean'. Fresh git worktree: run " "'python scripts/provision_worktree_data.py'. Fresh install: wait for " "the first daily refresh to build change_events.jsonl.", ) else: obs.info("delisting_inference_ready") except Exception as exc: # noqa: BLE001 — enrichment preflight, never block boot obs.error( "delisting_inference_preflight_failed", error_type=type(exc).__name__, error=str(exc), ) # Data plane first (synchronous): download and validate the latest candidate before prewarm, # but never overlay it onto serving data. Promotion is a reviewed deployment operation. # It is env-gated and fail-open; this guard ensures startup still cannot die here. if distribution() is Distribution.FULL: load_full_registration().boot_sync() # All selected warm work blocks readiness so customer requests cannot contend with CPU-heavy # cache construction. SINETTI_PREWARM_OPTIONAL=0 keeps critical + landing only; # SINETTI_PREWARM=0 (test suite) skips every phase. /api/ready still reports frontend.stale # from frontend_build_status independently of these warm phases. if os.environ.get("SINETTI_PREWARM", "1") != "0": _run_prewarm_phase("critical", _prewarm_critical) else: obs.info("prewarm_phase_skipped", phase="critical", reason="SINETTI_PREWARM=0") _prewarm_before_ready() # After every warm phase, before the first request: the warmed heap is the boot heap. _freeze_boot_heap() if distribution() is Distribution.FULL: stop_regulatory_scheduler = load_full_registration().start_regulatory_scheduler(app) else: stop_regulatory_scheduler = None from sinetti_services.ask_cohort_store import start_retention_worker stop_ask_retention = start_retention_worker() heartbeat = asyncio.create_task(obs.worker_heartbeat(), name="worker-heartbeat") try: yield finally: heartbeat.cancel() with suppress(asyncio.CancelledError): await heartbeat stop_ask_retention() if distribution() is Distribution.FULL: stop_regulatory_scheduler() def _default_static_dir() -> Path | None: from sinetti_services.frontend_build_status import served_dist_dir dist = served_dist_dir(Path(__file__).parents[1]) return dist if dist.exists() else None class _PersistenceSchemaError(RuntimeError): pass def _required_persistence_targets() -> dict[str, tuple[dict, ...]]: import enrich_store import state_paths import store_billing import store_iscc from sinetti_services import ai_meter from sinetti_services import data_access_meter from sinetti_services import ops_error_ledger from sinetti_services import ops_ownership intel_required_schema = dict(_INTEL_REQUIRED_SCHEMA) intel_required_indexes = set(_INTEL_REQUIRED_INDEXES) private_dialect = store_iscc._private_data_persistence_config().dialect if private_dialect == "postgres": # Residual hub SQLite after Track C: live certificate cache only. intel_required_schema = { "iscc_certs": _INTEL_REQUIRED_SCHEMA["iscc_certs"], } intel_required_indexes = {"ix_iscc_holder", "ix_iscc_country"} enrich_required_schema = dict(_ENRICH_REQUIRED_SCHEMA) if enrich_store._history_persistence_config().dialect == "postgres": # enrichment_cache + enrichment_history both live on the private plane. enrich_required_schema = {} targets = { "intel": ( { "path": Path(store_iscc.DB_PATH), "tables": intel_required_schema, "indexes": frozenset(intel_required_indexes), }, { "path": state_paths.intel_db_path(), "tables": enrich_required_schema, "indexes": frozenset(), }, ), "ai_meter": ( { "path": Path(ai_meter.DB_PATH), "tables": _AI_REQUIRED_SCHEMA, "indexes": _AI_REQUIRED_INDEXES, }, ), "data_access_meter": ( { "path": Path(data_access_meter.DB_PATH), "tables": _DATA_ACCESS_REQUIRED_SCHEMA, "indexes": _DATA_ACCESS_REQUIRED_INDEXES, }, ), "ops_error_ledger": ( { "path": Path(ops_error_ledger.DB_PATH), "tables": _OPS_ERROR_REQUIRED_SCHEMA, "indexes": _OPS_ERROR_REQUIRED_INDEXES, }, ), } if store_billing._private_data_persistence_config().dialect == "sqlite": import store_auth targets["auth"] = ( { "path": Path(store_auth.DB_PATH), "tables": _AUTH_REQUIRED_SCHEMA, "indexes": _AUTH_REQUIRED_INDEXES, }, ) targets["billing"] = ( { "path": Path(store_billing.DB_PATH), "tables": _BILLING_REQUIRED_SCHEMA, "indexes": frozenset(), }, ) targets["ops_ownership"] = ( { "path": Path(ops_ownership.DB_PATH), "tables": _OPS_OWNERSHIP_REQUIRED_SCHEMA, "indexes": _OPS_OWNERSHIP_REQUIRED_INDEXES, }, ) return targets def _probe_sqlite_schema( target: dict, *, busy_timeout_ms: int = PERSISTENCE_PROBE_LOCK_BUDGET_MS, ) -> None: path = Path(target["path"]).resolve() uri = f"{path.as_uri()}?mode=rw" connection = sqlite3.connect( uri, uri=True, timeout=busy_timeout_ms / 1000, ) try: connection.row_factory = sqlite3.Row connection.execute(f"PRAGMA busy_timeout = {busy_timeout_ms}") connection.execute("SELECT 1").fetchone() for table, required_columns in target["tables"].items(): actual_columns = { str(row["name"]) for row in connection.execute( "SELECT name FROM pragma_table_info(?)", (table,), ).fetchall() } missing_columns = set(required_columns) - actual_columns if missing_columns: raise _PersistenceSchemaError( f"{table} is missing columns: {', '.join(sorted(missing_columns))}" ) actual_indexes = { str(row["name"]) for row in connection.execute( "SELECT name FROM sqlite_master WHERE type = 'index'" ).fetchall() } missing_indexes = set(target["indexes"]) - actual_indexes if missing_indexes: raise _PersistenceSchemaError( f"missing indexes: {', '.join(sorted(missing_indexes))}" ) connection.execute("BEGIN IMMEDIATE") connection.execute( "CREATE TABLE IF NOT EXISTS __sinetti_write_probe (probe INTEGER NOT NULL)" ) connection.execute("INSERT INTO __sinetti_write_probe (probe) VALUES (1)") connection.rollback() finally: if connection.in_transaction: connection.rollback() connection.close() def _is_lock_contention(exc: BaseException) -> bool: """A busy/locked SQLite file — the app's own traffic — not a structural fault.""" if not isinstance(exc, sqlite3.OperationalError): return False message = str(exc).lower() return "locked" in message or "busy" in message def _probe_required_persistence( targets: dict[str, tuple[dict, ...]], contention_streaks: dict[str, int] | None = None, ) -> dict: checks: dict[str, dict[str, object]] = {} streaks = contention_streaks if contention_streaks is not None else {} started = time.perf_counter() for component, component_targets in targets.items(): try: for target in component_targets: spent_ms = (time.perf_counter() - started) * 1000 remaining_ms = PERSISTENCE_PROBE_LOCK_BUDGET_MS - spent_ms _probe_sqlite_schema( target, busy_timeout_ms=int(max(PERSISTENCE_PROBE_MIN_TIMEOUT_MS, remaining_ms)), ) except Exception as exc: if _is_lock_contention(exc): streak = streaks.get(component, 0) + 1 streaks[component] = streak tolerated = streak <= PERSISTENCE_PROBE_CONTENTION_TOLERANCE checks[component] = { "ok": tolerated, "required": True, "reason": "contended", } report = obs.warn if tolerated else obs.error report( "required_persistence_probe_contended" if tolerated else "required_persistence_probe_failed", component=component, error_type=type(exc).__name__, error=str(exc), consecutive=streak, tolerated=tolerated, ) else: streaks[component] = 0 reason = ( "schema incomplete" if isinstance(exc, _PersistenceSchemaError) else "unavailable" ) checks[component] = {"ok": False, "required": True, "reason": reason} obs.error( "required_persistence_probe_failed", component=component, error_type=type(exc).__name__, error=str(exc), ) else: streaks[component] = 0 checks[component] = {"ok": True, "required": True, "reason": ""} return { "ok": all(bool(check["ok"]) for check in checks.values()), "checks": checks, } def _heal_missing_required_columns(targets: dict[str, tuple[dict, ...]]) -> None: """Add any required column the initializers and targeted migrations left missing. The persistent bucket DBs carry years of drift; the 2026-08-01 deploy found shapes no code generation ever created fresh (a `dismissals` without `username`), so boot could finish green while the readiness probe stayed 'schema incomplete' forever. Healing is strictly additive — `ALTER TABLE ... ADD COLUMN` (nullable), never a drop or rewrite — and runs AFTER the targeted tenant migrations so it only patches residual drift. Missing tables and indexes stay the initializers' job (they already CREATE IF NOT EXISTS); a table absent here is left for the probe to report honestly. """ for component, component_targets in targets.items(): for target in component_targets: path = Path(target["path"]) if not path.is_file(): continue try: connection = sqlite3.connect(path) except Exception: # noqa: BLE001 - healing is best-effort; the probe reports truth continue try: connection.row_factory = sqlite3.Row for table, required_columns in target["tables"].items(): actual = { str(row["name"]) for row in connection.execute( "SELECT name FROM pragma_table_info(?)", (table,) ).fetchall() } if not actual: continue for column in sorted(set(required_columns) - actual): try: connection.execute( f'ALTER TABLE "{table}" ADD COLUMN "{column}" TEXT' ) obs.info( "required_schema_healed", component=component, table=table, column=column, ) except Exception as exc: # noqa: BLE001 - report, keep healing the rest obs.error( "required_schema_heal_failed", component=component, table=table, column=column, error=str(exc), ) connection.commit() finally: connection.close() def _initialize_required_persistence(app: FastAPI) -> None: checks: dict[str, dict[str, object]] = {} def run(component: str, initializer) -> None: try: initializer() except Exception as exc: checks[component] = { "ok": False, "required": True, "reason": "initialization failed", } obs.error( "required_persistence_initialization_failed", component=component, error_type=type(exc).__name__, error=str(exc), ) else: checks[component] = {"ok": True, "required": True, "reason": ""} def initialize_intel() -> None: import store_iscc store_iscc.init_iscc_tables() def initialize_ai_meter() -> None: from sinetti_services import ai_meter ai_meter.connect().close() def initialize_data_access_meter() -> None: from sinetti_services import data_access_meter data_access_meter.connect().close() def initialize_ops_error_ledger() -> None: from sinetti_services import ops_error_ledger ops_error_ledger.connect().close() def initialize_ops_ownership() -> None: from sinetti_services import ops_ownership ops_ownership.connect().close() def initialize_billing() -> None: import store_billing store_billing.init_billing_tables() store_billing.seed_billing_defaults() def initialize_auth() -> None: import store_auth store_auth.init_auth_tables() # Billing membership must exist before intel tenant migration stamps org_id. run("billing", initialize_billing) run("auth", initialize_auth) run("intel", initialize_intel) run("ai_meter", initialize_ai_meter) run("data_access_meter", initialize_data_access_meter) run("ops_error_ledger", initialize_ops_error_ledger) run("ops_ownership", initialize_ops_ownership) startup = { "ok": all(bool(check["ok"]) for check in checks.values()), "checks": checks, } targets = _required_persistence_targets() try: _heal_missing_required_columns(targets) except Exception as exc: # noqa: BLE001 - healing must never take the app down obs.error( "required_schema_heal_failed", component="*", error_type=type(exc).__name__, error=str(exc), ) app.state.required_persistence_startup = startup app.state.required_persistence = startup # Per-app, per-worker: how many consecutive probes each component has been locked for. contention_streaks: dict[str, int] = {} app.state.required_persistence_probe = lambda: _probe_required_persistence( targets, contention_streaks ) def _run_optional_startup_seeders() -> None: """Best-effort startup hydration after release resolution and table initialization.""" import iscc import store_iscc seeders = ( ("saved_views_seed", store_iscc.import_saved_views_seed), ("iscc_compliance_seed", iscc.first_run_load), ) for name, seeder in seeders: try: result = seeder() except Exception as exc: # noqa: BLE001 — optional hydration must never block V2 boot obs.error( "optional_startup_seed_failed", seed=name, error_type=type(exc).__name__, error=str(exc), ) else: details = {"imported": result} if isinstance(result, int) else {} obs.info("optional_startup_seed_complete", seed=name, **details) def _customer_router(router: APIRouter) -> APIRouter: """The router the active edition registers: as is in the full edition, else a copy without the trimmed operations (route_access.CUSTOMER_TRIMMED_OPERATIONS), such as the NEa line of the holder router. FastAPI includes routers by reference, so the shared module router itself is never altered and a full app built in the same process keeps every route. """ if distribution() is Distribution.FULL: return router from fastapi.routing import APIRoute from sinetti_api.route_access import CUSTOMER_TRIMMED_OPERATIONS if router.dependencies or router.on_startup or router.on_shutdown: raise RuntimeError("customer route trim copies plain routers only") kept = APIRouter() kept.routes.extend( route for route in router.routes if not ( isinstance(route, APIRoute) and any((method, route.path) in CUSTOMER_TRIMMED_OPERATIONS for method in route.methods) ) ) return kept def create_app(static_dir: str | Path | None = None) -> FastAPI: # The fixed full composition requires its private add-on before mounting any route. if distribution() is Distribution.FULL: operator = load_full_registration() else: operator = None from sinetti_api.runtime_security import hosted_runtime, local_loopback_request from sinetti_api.browser_session import BrowserSessionMiddleware hosted = hosted_runtime() app = FastAPI( title="Sinetti API", version="2.0.0", lifespan=_lifespan, docs_url=None if hosted else "/docs", redoc_url=None if hosted else "/redoc", openapi_url=None if hosted else "/openapi.json", ) # Compress responses: the certificate/holder/analytics JSON is large and highly repetitive # (identical keys per row), so gzip shrinks it ~90%. Biggest single network win for Browse. app.add_middleware(GZipMiddleware, minimum_size=1000) app.add_middleware(BatchCheckUploadLimitMiddleware) app.add_middleware(BrowserSessionMiddleware) @app.middleware("http") async def request_observability(request, call_next): # A caller-supplied X-Request-ID is kept only when it looks like an id; anything else # would put arbitrary text into logs and the error ledger (diagnostics_contract). trace_id = _diagnostics.safe_trace_id(request.headers.get("X-Request-ID")) request.state.trace_id = trace_id started = time.perf_counter() if request.url.path in {"/docs", "/docs/oauth2-redirect", "/redoc", "/openapi.json"} and not local_loopback_request(request): response = JSONResponse(status_code=404, content={"ok": False, "message": "Not found"}) else: response = await call_next(request) response.headers["X-Request-ID"] = trace_id response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "DENY" response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()" if hosted_runtime() or request.url.scheme == "https": response.headers["Strict-Transport-Security"] = "max-age=31536000" response.headers["Content-Security-Policy"] = ( "default-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; " "script-src 'self'; style-src 'self' 'unsafe-inline'; " # MapLibre GL spawns its render worker from a blob: URL. Without this the # worker throws a SecurityError and the Browse error boundary swallows the # whole page. Scoped to worker-src so script-src stays 'self' only. "worker-src 'self' blob:; " # The intro video is fetched with the session's Bearer header (a