Spaces:
Running
Running
Download sinetti_api/app.py from Couscouskabyle/iscc-app: direct link, hf CLI and curl.
- Browser
- Download file 75.3 kB
-
https://huggingface.co/spaces/Couscouskabyle/iscc-app/resolve/main/sinetti_api/app.py
- Command line
-
hf download hf://spaces/Couscouskabyle/iscc-app/sinetti_api/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Couscouskabyle/iscc-app/resolve/main/sinetti_api/app.py
75.3 kB
from __future__ import annotations
import os
import asyncio
import gc
import sqlite3
import sys
import time
import uuid
from collections.abc import Callable
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
from sinetti_api.distribution import Distribution, distribution, load_full_registration
if distribution() is Distribution.FULL:
import secrets_local
from sinetti_services import serving_recovery
import obs
import state_paths
from sinetti_api.addons import batch_check, market_balances
from sinetti_api.upload_limits import BatchCheckUploadLimitMiddleware
if distribution() is Distribution.FULL:
# Mirror Classic (app.py): load AI/search keys from a local .env for APP_BACKEND=local runs.
# Gated + no-ops on HF (existing env / Space Secrets win). Without this, V2 uvicorn starts
# with a bare environment and Sentinel/chat see a missing ANTHROPIC_API_KEY even though .env
# has it. The fixed CUSTOMER composition does not ship the desktop dotenv helper.
secrets_local.load_local_dotenv()
# Private mode (Private Cloud): arm the outbound-connection guard before anything - boot data
# sync, prewarm, any router - can open a socket. No-op unless SINETTI_PRIVATE_MODE=1.
from sinetti_services import egress_guard as _egress_guard
_egress_guard.install()
from sinetti_services import diagnostics_contract as _diagnostics # noqa: E402
from sinetti_api.deps import (
require_admin,
require_data_access,
require_feature,
require_research_data,
research_scope,
require_user,
)
from sinetti_api.errors import (
ActionUnavailable,
FeatureBlocked,
action_unavailable_handler,
feature_blocked_handler,
http_exception_handler,
request_validation_exception_handler,
tenant_required_handler,
unhandled_exception_handler,
)
from sinetti_services.tenant_context import TenantRequired
from sinetti_api.routers import (
about,
access_public,
ask_conversations,
discussions,
analytics,
archive_life,
auth,
business_footprint,
capabilities,
category_timelines,
certificates,
certification_bodies,
chat,
contact,
feedback,
counterparty,
dossier,
email_public,
feedstocks,
flags,
fuel_conversion,
fuel_specs,
intro_video,
fuels,
geography,
health,
holders,
maps,
material_families,
me_profile,
monitor,
narrative,
notifications,
ops,
places,
product_signals,
risk,
saf_exports,
activity,
saved_views,
search,
security,
sentinel,
site_footprint,
site_assessment,
soft_presets,
timeline,
tours,
watchlist,
)
# Kept in sync with sinetti_api/routers/feedstocks.py's _PATHWAY_FAMILIES so a future family
# added to the router is automatically prewarmed without touching this module.
_PREWARM_FAMILIES = tuple(sorted(feedstocks._PATHWAY_FAMILIES))
# Once the shared lock budget below is spent, every remaining target waits only this long --
# enough to take an uncontended lock, not enough to add another wait per component. Measured
# 2026-08-13: with a 250 ms floor instead, one contended readiness poll spent ~250-800 ms on
# EACH of the seven targets in turn and /api/ready peaked at 3,584 ms, against a 1 s budget
# that was only ever enforced on the first target.
PERSISTENCE_PROBE_MIN_TIMEOUT_MS = 25
# A locked SQLite file is not the same fault as a missing table. The lock is held by this
# app's own in-flight write and clears in milliseconds; the platform's remedy for an
# unready container -- restart it, or take it out of rotation -- clears nothing and moves
# that traffic onto the sibling worker, which shares the same file. So the probe waits for
# a lock it can still get instead of reporting the app unhealthy for serving customers.
# The budget is shared across every target so /api/ready stays inside the poll timeouts
# that scripts/preflight_deploy.py and scripts/assert_v2_serve_matches_desk.ps1 already use.
PERSISTENCE_PROBE_LOCK_BUDGET_MS = 1000
# Waiting alone would only move the brittleness: a wedged writer would still flip the
# container unhealthy, just later. A lock that survives this many consecutive readiness
# probes has stopped being contention and IS a not-ready condition, so the signal survives.
PERSISTENCE_PROBE_CONTENTION_TOLERANCE = 1
_INTEL_REQUIRED_SCHEMA = {
"iscc_certs": frozenset({
"certificate_id", "holder_name", "holder_norm", "country", "scope", "proc_unit_type",
"feedstocks", "add_ons", "products", "valid_from", "valid_until", "suspended_date",
"issuing_cb", "raw_status",
}),
"iscc_sync_log": frozenset({
"id", "synced_at", "n_total", "n_new", "n_withdrawn", "n_renewed", "deltas_json",
"source",
}),
"iscc_compliance_history": frozenset({"id", "snapshot_at", "source", "kind", "members_json"}),
"iscc_tracked": frozenset({
"org_id", "user_id", "holder_norm", "label", "pinned_at", "list_name", "alerts",
}),
"iscc_saved_views": frozenset({"org_id", "user_id", "scheme", "name", "payload", "created_at"}),
"iscc_saved_view_shares": frozenset({
"share_id", "created_by_user_id", "created_at", "expires_at",
"suggested_name", "scheme", "payload",
}),
"iscc_feedstock_alias": frozenset({"feedstock", "alias"}),
"iscc_timeline_sets": frozenset({"user_id", "name", "scheme", "holder_norms", "created_at"}),
"holder_bd_research": frozenset({
"org_id", "holder_norm", "scheme", "holder_name", "country", "researched_at",
"search_provider", "search_query_count", "ai_provider", "report_json", "evidence_json",
"status", "error",
}),
"producer_cert_links": frozenset({"producer_id", "certificate_id"}),
"iscc_alerts": frozenset({"id", "holder_norm", "dedup_key", "kind", "title", "detail", "created_at"}),
"dismissals": frozenset({"org_id", "user_id", "dedup_key", "dismissed_at"}),
"notification_preferences": frozenset({
"org_id", "user_id", "delivery_mode", "destination", "destination_verified",
"unsubscribed", "updated_at",
}),
"user_profile": frozenset({"org_id", "user_id", "work_role", "updated_at"}),
"notification_delivery_attempts": frozenset({
"idempotency_key", "org_id", "user_id", "destination", "event_ids_json", "final_status",
"attempt_count", "claim_generation", "created_at", "updated_at", "attempted_at",
"next_attempt_at", "provider_response_id",
}),
"notification_event_claims": frozenset({
"org_id", "user_id", "destination_key", "event_id", "attempt_idempotency_key",
"claimed_at", "updated_at",
}),
"web_push_subscriptions": frozenset({
"subscription_id", "org_id", "user_id", "endpoint", "p256dh", "auth",
"created_at", "updated_at", "disabled_at",
}),
"web_push_delivery_claims": frozenset({
"subscription_id", "event_id", "org_id", "user_id", "state", "claimed_at",
"updated_at", "provider_status",
}),
"web_push_delivery_queue": frozenset({
"subscription_id", "event_id", "org_id", "user_id", "holder_norm", "scheme",
"enqueued_at", "next_attempt_at",
}),
"login_lockout": frozenset({"username", "fail_count", "locked_until"}),
}
_INTEL_REQUIRED_INDEXES = frozenset({
"ix_notification_attempts_user",
"ix_notification_attempts_retry",
"ix_notification_event_claims_attempt",
"ix_web_push_subscriptions_tenant",
"ix_web_push_delivery_claims_tenant",
"ix_web_push_delivery_queue_oldest",
"ix_web_push_delivery_queue_tenant",
"ix_web_push_delivery_queue_ready",
"ix_iscc_holder",
"ix_iscc_country",
})
_ENRICH_REQUIRED_SCHEMA = {
"enrichment_cache": frozenset({
"provider", "input_key", "status", "payload_json", "fetched_at", "expires_at",
}),
"enrichment_history": frozenset({
"id", "org_id", "holder_norm", "scheme", "verdict_level", "reasons_json",
"produced_at", "produced_by",
}),
}
_AI_REQUIRED_SCHEMA = {
"ai_usage": frozenset({
"id", "user_id", "feature", "model", "input_tokens", "output_tokens",
"cache_write_tokens", "cache_read_tokens", "search_queries", "cost_usd", "created_at",
}),
"ai_limits": frozenset({"user_id", "feature", "monthly_usd", "updated_at", "updated_by"}),
"ai_reservations": frozenset({
"reservation_id", "user_id", "feature", "reserved_usd", "created_at", "expires_at",
}),
}
_AI_REQUIRED_INDEXES = frozenset({"idx_ai_usage_user_month", "idx_ai_reservations_user_feature"})
_DATA_ACCESS_REQUIRED_SCHEMA = {
"data_access_limits": frozenset({"user_id", "requests_per_window", "daily_export_rows", "monthly_export_rows", "periodic_unlimited", "updated_at", "updated_by"}),
"data_request_events": frozenset({"event_id", "user_id", "occurred_at"}),
"data_access_org_limits": frozenset({"org_id", "requests_per_window", "daily_export_rows", "monthly_export_rows", "periodic_unlimited", "updated_at", "updated_by"}),
"data_org_request_events": frozenset({"event_id", "org_id", "occurred_at"}),
"data_export_audit": frozenset({"audit_id", "trace_id", "created_at", "user_id", "org_id", "is_admin", "method", "route_name", "schemes_json", "filters_json", "exact_selection_count", "exact_selection_sha256", "row_count", "byte_count", "proposed_filename", "outcome", "reason"}),
"data_limit_changes": frozenset({
"change_id", "target_kind", "target_id", "changed_at", "changed_by",
"old_settings_json", "new_settings_json",
}),
}
_DATA_ACCESS_REQUIRED_INDEXES = frozenset({
"data_request_events_user_time_idx",
"data_org_request_events_org_time_idx",
"data_export_audit_time_idx",
"data_export_audit_user_time_idx",
"data_export_audit_org_time_idx",
"data_limit_changes_target_idx",
})
_AUTH_REQUIRED_SCHEMA = {
# Exactly what session validation reads: a drifted auth store silently turns every
# issued token into a guest (validate() swallows store errors), which is invisible to
# every gate until the live smoke fails β probe it like the rest.
"auth_sessions": frozenset({
"session_digest", "user_id", "auth_version", "issued_at", "last_seen_at",
"expires_at", "device_label", "revoked_at", "revoked_by", "revoke_reason",
}),
"auth_account_controls": frozenset({
"user_id", "state", "auth_version_delta", "changed_at", "changed_by", "reason",
}),
}
_AUTH_REQUIRED_INDEXES = frozenset({"auth_sessions_user_active_idx"})
_OPS_ERROR_REQUIRED_SCHEMA = {
"ops_error_events": frozenset({
"event_id", "created_at", "source", "trace_id", "user_id",
"path", "component", "message", "fingerprint",
}),
}
_OPS_ERROR_REQUIRED_INDEXES = frozenset({
"ops_error_events_created_idx",
"ops_error_events_source_created_idx",
"ops_error_events_fingerprint_idx",
})
_OPS_OWNERSHIP_REQUIRED_SCHEMA = {
"ops_ownership_events": frozenset({
"event_id", "created_at", "actor_user_id", "org_id", "action", "detail",
}),
}
_OPS_OWNERSHIP_REQUIRED_INDEXES = frozenset({
"ops_ownership_events_created_idx",
"ops_ownership_events_org_created_idx",
})
_BILLING_REQUIRED_SCHEMA = {
"billing_orgs": frozenset({
"org_id", "name", "status", "seats", "stripe_customer_id", "created_at", "updated_at",
}),
"billing_org_members": frozenset({"org_id", "user_id", "added_at"}),
"billing_plans": frozenset({
"plan_id", "name", "features", "schemes", "seat_limit", "ai_budgets", "created_at", "updated_at",
}),
"billing_plan_assignments": frozenset({
"org_id", "plan_id", "current_period_start", "current_period_end", "source", "updated_at", "updated_by",
}),
}
def _prewarm_critical() -> Exception | None:
"""Critical readiness warm: corpus + Browse + pathway index + landing feedstock cards.
This is the path the Browse landing page and feedstock-map capacity probe hit first.
Gzipped World-list warm alone does not cover POST /api/certificates/query. Pathways + the
certificate-pathway index are warmed here so card builds do not pay a cold index, and the
first page of default World Browse holders get their feedstock-card responses built so the
first real fan-out is a cache hit. Best-effort: failures fall through to the normal
on-request load path.
"""
try:
import iscc
import sinetti_data
from sinetti_api.routers import certificates as _certs
from sinetti_api.routers import counterparty as _counterparty
from sinetti_api.routers import holders as _holders
from sinetti_services import pathways_service
from sinetti_services.holder_name_index import holder_name_index_for
from sinetti_services.read_models import list_certificate_summaries
groups = iscc._all_scheme_certs()
_counterparty.prewarm_fit_index(groups)
list_certificate_summaries(groups, scheme="World", status="all")
# Browse/Verify typeahead: prefix index over raw rows (avoids cold first keystroke).
holder_name_index_for(groups, scheme="World")
_certs.prewarm_default_browse_query(groups)
sinetti_data.load_app_pathways()
from sinetti_data_reads.curated_results import customer_mode
if not customer_mode():
# Customer pathway indexes live on each bound result snapshot, built on first read.
pathways_service._certificate_pathway_index()
_holders.prewarm_default_browse_feedstock_cards(groups)
# Every page loads /api/material-families; cold it built the payload on the first request.
from sinetti_services import family_resolution
family_resolution.build_families_payload_bytes()
audit_failure = _prewarm_audit_details()
if audit_failure is not None:
obs.warn("prewarm_audit_details_failed", error=type(audit_failure).__name__)
except Exception as exc:
return exc
return None
def _prewarm_audit_details() -> Exception | None:
"""Unpack the certificate-PDF details file before the first card asks for an audit.
Each Browse card fetches its audits; cold, the first ones unpacked the 1.2 GB details copy
and the whole server waited (SIN-0996). The customer edition reads released results instead
and its provider returns None here.
"""
try:
from sinetti_services import documents_source
documents_source.audit_details_path()
except Exception as exc:
return exc
return None
def _prewarm_search() -> Exception | None:
"""Build the name-search indexes so the first text search after boot is fast.
Verify (full search) and Browse each match through a token index built once per corpus
(sinetti_domain.name_search.NameSearchIndex, a few seconds cold). The default Browse probe can
resolve to suggestions before the filter needs its index, so explicitly build the canonical
World summary index after that probe. Called after selected startup phases so later warm work
cannot evict these bounded indexes before readiness.
"""
try:
import iscc
from sinetti_domain.metrics import deduplicate_registry_rows
from sinetti_domain.name_search import name_search_index_for
from sinetti_services.search_service import search_omnibox
from sinetti_services.read_models import list_certificate_summaries
from sinetti_api.routers import certificates as _certs
groups = iscc._all_scheme_certs()
search_omnibox(groups, "prewarm", scheme="World")
# Keep the ordinary Browse query warm, then materialize the exact index its canonical
# World/all filter uses, including when the dummy holder probe resolves to suggestions.
_certs.prewarm_default_browse_query(groups, holder="prewarm")
canonical_rows = deduplicate_registry_rows(
list_certificate_summaries(groups, scheme="World", status="all")
)
name_search_index_for(canonical_rows, aliases_field="entity_aliases")
except Exception as exc:
return exc
return None
def _prewarm_optional() -> Exception | None:
"""Best-effort deep-dive / sites / list-bytes / CB readiness warm.
Skipped only when SINETTI_PREWARM_OPTIONAL=0 (or SINETTI_PREWARM=0). Must build under the
EXACT memo key real requests use (sinetti_api/routers/feedstocks.py), or this warms a key
no request ever reads. That means: the same expiry_days/delisted_ids as
get_feedstock_context() (build_family_deepdive's memo key includes int(expiry_days), and
defaults to 0 if omitted -- a permanent miss against the real default of 90), and passing
`groups` into build_fwo_deepdive (by_scheme=None short-circuits its memo write entirely).
"""
try:
import iscc
import sinetti_data
import sinetti_services.deepdive_service as deepdive_service
from sinetti_api.deps import get_feedstock_context
from sinetti_services.feedstock_deepdive import build_feedstock_deepdive
groups = iscc._all_scheme_certs()
sinetti_data.load_app_pathways()
sinetti_data.load_app_sites()
context = get_feedstock_context()
# Site markers first: Browse (Source near, card framing), Producers and Certified Sites
# all wait on /api/maps/sites, and after the deep-dives below it stayed cold for the
# first minute of traffic after a restart (11-26 s cold, 2026-09-30).
_prewarm_site_markers(groups, context)
_prewarm_near_measurement(groups)
for family in _PREWARM_FAMILIES:
deepdive_service.build_family_deepdive(
family,
groups,
scheme="World",
status="active",
expiry_days=context["expiry_days"],
delisted_ids=context["delisted_ids"],
)
# supply_chain is attached on every family response and has its own memo β
# warm the default active-status key the Feedstocks page requests.
build_feedstock_deepdive(
family,
groups,
filters={"statuses": ["active"]},
expiry_days=context["expiry_days"],
)
deepdive_service.build_fwo_deepdive(groups, scheme="World", status="active")
# Fuels: /api/fuels/{family}/deepdive serves biodiesel/ethanol/biogas through this SAME
# build_family_deepdive memo, but _PREWARM_FAMILIES lists only feedstock families, so
# those three were never warmed anywhere. Measured cost of a cold one on the 2026-08
# corpus: 60-67 s on an idle machine, which is what timed out deepdive-evolution in the
# release gate and is what the first user after a Space boot would have paid. HVO is
# already covered by _prewarm_landing_deepdive. The route passes empty regions/countries
# sets and the memo key normalises `regions or ()`, so omitting them hits the same key.
from sinetti_api.routers.fuels import _FUEL_FAMILIES
for fuel in _FUEL_FAMILIES:
if fuel == "hvo":
continue
deepdive_service.build_family_deepdive(
fuel,
groups,
scheme="World",
status="active",
expiry_days=context["expiry_days"],
delisted_ids=context["delisted_ids"],
)
# #62: warm the memoized+gzipped World/all list bytes too, so the first Browse paint after
# a cold start is served from cache instead of paying the full build+encode+compress cost.
from sinetti_api.routers import certificates as _certs
_certs._gzipped_list_bytes(groups, "World", "all")
# CB deep-dive: warm certificate rows + facts cube + year series (gallery cold path ~20s).
sinetti_data.load_app_certificate_rows()
from sinetti_api.routers import certification_bodies as _cb
_cb.prewarm_cb_deepdive()
# Geography: warm the world matrix (~2 s cold) β it is the deep dive's default view.
from sinetti_api.routers import geography as _geo
_geo.prewarm_geography_deepdive()
_prewarm_map_overlays()
_prewarm_holder_page()
_prewarm_ask_library()
except Exception as exc:
return exc
return None
def _prewarm_site_markers(groups: dict[str, list[dict]], context: dict) -> None:
"""Build the site markers and the gzipped full-plan /api/maps/sites body under route keys."""
from datetime import date
import sinetti_data
from sinetti_services.map_service import list_site_markers
site_markers = list_site_markers(
sinetti_data.load_app_sites(),
certs=groups,
today=date.today().isoformat(),
expiry_days=context["expiry_days"],
delisted_ids=context["delisted_ids"],
)
# The serialized body has its own guard: a bad marker must not cost the warms after it (the
# request path fails open to an empty envelope on the same error).
try:
from sinetti_api.routers import maps as _maps
_maps.warm_site_markers_payload(site_markers)
except Exception:
obs.warn("prewarm_site_markers_payload_failed")
def _prewarm_browse_maps() -> Exception | None:
"""Warm what the first filtered Browse page builds besides its list (SIN-0995).
A filtered Browse page fires the list query, two /api/maps/holder-coords/query calls and
/api/maps/sites at once. Cold, those builds plus the delisted-id scan and the first pathway
material scan shared one interpreter lock: the UCO + Europe list answered after 60-110 s
against 2 s alone (2026-10-25). Runs before bind, so on the :8511 proxy the old backend keeps
serving until this one is warm.
"""
try:
import iscc
from sinetti_api.deps import get_feedstock_context
from sinetti_api.routers import certificates as _certs
from sinetti_api.routers import maps as _maps
from sinetti_services import map_service
groups = iscc._all_scheme_certs()
context = get_feedstock_context()
_prewarm_site_markers(groups, context)
map_service.warm_holder_coords(groups)
map_service.warm_browse_country_centres()
_maps._sibling_norms(groups)
# One real material-filtered query: entitled pathway rows, unit evidence and the
# material label memo every feedstock filter reads.
_certs.prewarm_default_browse_query(groups, feedstocks=["Used cooking oil (UCO)"])
except Exception as exc:
return exc
return None
def _prewarm_near_measurement(groups: dict[str, list[dict]]) -> None:
"""Build the indexes behind every "near" measurement (Browse Source near, the map's ring).
The postal locality index, the physical-site points and the per-scheme register points took
~9 s on the first Source query after a restart (2026-10-11), paid inside that request. They
memoise per scheme name, never per caller, so one measurement over the whole corpus through
the real path warms the keys every entitled subset reads. A request abandoned mid-build keeps
building them on purpose (SIN-0150), so warming here is what takes them off the request path.
"""
from sinetti_services import browse_near
from sinetti_services.read_models import list_certificate_summaries
browse_near.holder_distances(
list_certificate_summaries(groups, scheme="World", status="all"),
lat=0.0,
lon=0.0,
radius_km=browse_near.normalize_radius_km(None),
allowed_schemes=groups,
)
def _prewarm_map_overlays() -> None:
"""Warm the Browse map overlays the first page load fetches alongside the certificate list.
The hosted run log of 2026-09-29 showed these as the slowest first calls after a restart:
JSON SOURCE producers 19.6 s (28 s cold on the desk: it builds the scheme="All" producer
markers it suppresses against) and ISCC life bounds 22 s (3.5 s alone). Both memoise per
process, so building them here makes the first customer request a cache hit. Called with
no arguments, exactly like their routes, so the warm lands on the key requests read.
"""
from sinetti_services import sites_source
from sinetti_services.iscc_life_bounds import get_iscc_life_bounds
get_iscc_life_bounds()
sites_source.json_source_markers()
sites_source.research_gap_markers()
if sites_source.mode() == sites_source.MODE_RELEASED:
# The seam already logged one ERROR line if the file is unusable; this is the one
# INFO line an operator reads to know which release scope is being served.
obs.info("released_sites_status", **sites_source.status())
def _prewarm_holder_page() -> None:
"""Warm the per-process caches every holder dossier reads, whichever holder it is.
Cold profile of 2026-09-30 (Neste dossier right after a restart, 2.3 s against 0.4 s warm):
the site-country locator behind register sites took ~1.1 s to build over every site row,
and the related-company registry files ~0.3 s to read. Both memoise per process.
"""
from sinetti_services import site_country
from sinetti_data_reads.curated_results import customer_mode as _customer_results_mode
if _customer_results_mode():
from sinetti_data_reads import company_results as company_external_relations
else:
from sinetti_services import company_external_relations
site_country.default_locate()
company_external_relations.load_registry_decisions()
def _prewarm_ask_library() -> None:
"""Build Ask's per-entitlement retrieval views (SIN-0572) for the plans in use.
Every authenticated question filters the library by the asker's schemes; a cold view costs
~1.3 s on the first question per plan. Local only: no question is embedded.
"""
import rag_store
from sinetti_services import entitlements
rag_store.prewarm(entitlements.ask_scheme_sets())
def _prewarm_uco_deepdive() -> Exception | None:
"""Warm only the default Feedstocks (UCO) family and supply-chain payloads."""
try:
import iscc
import sinetti_services.deepdive_service as deepdive_service
from sinetti_api.deps import get_feedstock_context
from sinetti_services.feedstock_deepdive import build_feedstock_deepdive
groups = iscc._all_scheme_certs()
context = get_feedstock_context()
deepdive_service.build_family_deepdive(
"uco",
groups,
scheme="World",
status="active",
expiry_days=context["expiry_days"],
delisted_ids=context["delisted_ids"],
)
build_feedstock_deepdive(
"uco",
groups,
filters={"statuses": ["active", "expiring"]},
expiry_days=context["expiry_days"],
)
except Exception as exc:
return exc
return None
def _prewarm_landing_deepdive() -> Exception | None:
"""Warm default Feedstocks (UCO) + Biofuels (HVO) landings before readiness.
The first open of Feedstocks always lands on UCO; without this warm the user pays a cold
~3 s family build + ~1 s supply_chain build. Biofuels lands on HVO; family memo alone is not
enough β build_fuels_deepdive still pays pathway rescans + crosstab (~2 s) unless the full
fuels payload is warmed. The full multi-family warm defaults on and can be skipped with
SINETTI_PREWARM_OPTIONAL=0.
"""
try:
failure = _prewarm_uco_deepdive()
if isinstance(failure, Exception):
return failure
import iscc
import sinetti_services.fuels_deepdive as fuels_deepdive
from sinetti_api.deps import get_feedstock_context
groups = iscc._all_scheme_certs()
context = get_feedstock_context()
fuels_deepdive.build_fuels_deepdive(
groups,
scheme="World",
status="active",
expiry_days=context["expiry_days"],
delisted_ids=context["delisted_ids"],
)
except Exception as exc:
return exc
return None
def _prewarm_data() -> None:
"""Full warm (critical Browse/summary + optional deep-dives). Used by e2e sync path."""
_run_prewarm_phase("critical", _prewarm_critical)
_run_prewarm_phase("landing", _prewarm_landing_deepdive)
_run_prewarm_phase("browse", _prewarm_browse_maps)
_run_prewarm_phase("optional", _prewarm_optional)
_run_prewarm_phase("search", _prewarm_search)
def _run_prewarm_phase(
phase: str, target: Callable[[], Exception | None]
) -> None:
"""Run one readiness warm best-effort and expose its startup cost."""
started = time.perf_counter()
obs.info("prewarm_phase_begin", phase=phase)
try:
failure = target()
except Exception as exc: # noqa: BLE001 β warm misses fall back to request-time builds
failure = exc
duration_s = round(time.perf_counter() - started, 3)
if isinstance(failure, Exception):
obs.error(
"prewarm_phase_failed",
phase=phase,
duration_s=duration_s,
error_type=type(failure).__name__,
error=str(failure),
)
else:
obs.info(
"prewarm_phase_done",
phase=phase,
duration_s=duration_s,
)
def _prewarm_before_ready() -> None:
"""Synchronously warm selected deep dives before the application reports ready.
The broad optional warm defaults on because its cold misses measured up to 60-80 seconds on
the 2026-08 corpus. SINETTI_PREWARM_LANDING=0 skips the UCO/fuels landing warm;
SINETTI_PREWARM_BROWSE=0 skips the first-Browse map warm; SINETTI_PREWARM_OPTIONAL=0 skips the
remaining broad phase but keeps the final Verify/Browse search warm; SINETTI_PREWARM=0 disables
warming here and the critical phase in the lifespan. Local serve scripts skip landing+optional
so bind is not blocked for minutes; HF/prod keep the defaults.
"""
if os.environ.get("SINETTI_PREWARM", "1") == "0":
obs.info("prewarm_before_ready_skipped", reason="SINETTI_PREWARM=0")
return
landing_disabled = os.environ.get("SINETTI_PREWARM_LANDING", "1") == "0"
browse_enabled = os.environ.get("SINETTI_PREWARM_BROWSE", "1") != "0"
if landing_disabled:
obs.info("prewarm_phase_skipped", phase="landing", reason="SINETTI_PREWARM_LANDING=0")
else:
_run_prewarm_phase("landing", _prewarm_landing_deepdive)
if not browse_enabled:
obs.info("prewarm_phase_skipped", phase="browse", reason="SINETTI_PREWARM_BROWSE=0")
else:
_run_prewarm_phase("browse", _prewarm_browse_maps)
if os.environ.get("SINETTI_PREWARM_OPTIONAL", "1") == "0":
obs.info("prewarm_phase_skipped", phase="optional", reason="SINETTI_PREWARM_OPTIONAL=0")
else:
_run_prewarm_phase("optional", _prewarm_optional)
_run_prewarm_phase("search", _prewarm_search)
# Collector settings applied once the boot heap is warm (SIN-0486). A full pass over the warmed
# heap (~9.3 million tracked objects) held the interpreter lock for 4-5 s on the owner PC, longer
# on the Space's shared cores, so uvicorn's 5 s worker health check killed the worker. Freezing
# the boot heap removes it from every later pass; the raised gen-0 threshold makes young passes
# rarer. Measured against a burst of 75 cold Browse searches.
GC_THRESHOLDS = (50_000, 20, 1_000)
# Any collector pass at least this long is logged with its generation and duration.
GC_SLOW_PASS_SECONDS = 0.5
_gc_pass_started_at = 0.0
# Bound once: the collector fires inside any request, so the callback must not share the
# module-level ``time`` that request timing (and its tests) read.
_gc_clock = time.perf_counter
def _log_slow_gc_pass(phase: str, info: dict) -> None:
"""gc.callbacks hook: time every pass, log only the slow ones (diagnostics, never raises)."""
global _gc_pass_started_at
if phase == "start":
_gc_pass_started_at = _gc_clock()
return
duration = _gc_clock() - _gc_pass_started_at
if duration < GC_SLOW_PASS_SECONDS:
return
try:
obs.warn(
"gc_pass_slow",
pid=os.getpid(),
generation=info.get("generation"),
duration_ms=round(duration * 1000),
collected=info.get("collected"),
frozen_objects=gc.get_freeze_count(),
)
except Exception: # noqa: BLE001 β a logging failure must not escape into the collector
pass
def _install_gc_pause_logger() -> None:
if _log_slow_gc_pass not in gc.callbacks:
gc.callbacks.append(_log_slow_gc_pass)
def _freeze_boot_heap() -> None:
"""Collect once, freeze the warmed boot heap and raise the collector thresholds.
SINETTI_GC_FREEZE=0 disables it. Skipped with SINETTI_PREWARM=0 too: without a warm there is
no boot heap worth freezing, and the test suite would pay a full collection per app start.
"""
if os.environ.get("SINETTI_GC_FREEZE", "1") == "0":
obs.info("gc_freeze_skipped", reason="SINETTI_GC_FREEZE=0")
return
if os.environ.get("SINETTI_PREWARM", "1") == "0":
obs.info("gc_freeze_skipped", reason="SINETTI_PREWARM=0")
return
started = time.perf_counter()
collected = gc.collect()
collect_ms = round((time.perf_counter() - started) * 1000)
gc.freeze()
gc.set_threshold(*GC_THRESHOLDS)
frozen = gc.get_freeze_count()
obs.info(
"gc_boot_heap_frozen",
pid=os.getpid(),
collect_ms=collect_ms,
collected=collected,
frozen_objects=frozen,
tracked_objects=frozen + len(gc.get_objects()),
thresholds=list(GC_THRESHOLDS),
)
# Python's default 5 ms GIL switch interval starves I/O-bound threads behind CPU-bound ones: a
# thread that releases the GIL for a SQLite row step or a stat() waits up to the whole interval to
# get it back. A cold RSO deep-dive next to one CPU-busy request went 4.0 s -> 80.7 s, and three
# heavy requests on a cold desk took 19-22 minutes each (2026-10-17). 0.5 ms measured 80.7 -> 23.2 s.
GIL_SWITCH_INTERVAL_SECONDS = 0.0005
def _shorten_gil_switch_interval() -> None:
"""Set the process GIL switch interval. SINETTI_GIL_SWITCH_INTERVAL overrides it (seconds);
0 keeps the interpreter default."""
raw = os.environ.get("SINETTI_GIL_SWITCH_INTERVAL", "").strip()
try:
interval = float(raw) if raw else GIL_SWITCH_INTERVAL_SECONDS
except ValueError:
interval = GIL_SWITCH_INTERVAL_SECONDS
if interval <= 0:
obs.info("gil_switch_interval_kept", seconds=sys.getswitchinterval())
return
sys.setswitchinterval(interval)
obs.info("gil_switch_interval_set", seconds=sys.getswitchinterval())
@asynccontextmanager
async def _lifespan(app: FastAPI):
obs.enable_worker_faulthandler()
_install_gc_pause_logger()
_shorten_gil_switch_interval()
recovery = serving_recovery.recover_pending_activation(
data_dir=state_paths.BUNDLED_DATA_DIR
)
obs.info("data_release_promotion_recovery", **recovery)
if recovery.get("status") not in ("clean", "restored"):
raise RuntimeError(
str(recovery.get("error") or "serving release promotion recovery failed")
)
import release_store
heal = release_store.boot_self_heal()
obs.info("release_store_boot", **heal)
import sinetti_data
sinetti_data.active_artifact_path.cache_clear() # resolve fresh after any self-heal switch
from sinetti_data_reads.curated_results import customer_mode
if customer_mode():
from sinetti_data_reads.material_results import install_feedstock_providers
install_feedstock_providers()
resolution = release_store.resolve_active()
obs.info("serving_resolution", **{k: str(v) for k, v in resolution.items()})
if not Path(resolution["sqlite_path"]).is_file():
# Resolution succeeded (never raises) but the file it points at doesn't exist. The most
# common cause: a fresh `git worktree add` never receives the gitignored serving data
# pair (see release_store.PAIR_NAMES) β every read silently returns [] (sinetti_data.
# _read_rows) instead of failing, so endpoints look "empty" rather than broken. This
# never blocks boot: prod always has the bundled file, and pytest/CI intentionally
# exercise the empty-corpus fallback.
obs.error(
"serving_artifact_missing",
sqlite_path=str(resolution["sqlite_path"]),
source=resolution["source"],
reason=resolution["reason"],
hint="corpus data file not found on disk -- if this is a fresh git worktree, run "
"'python scripts/provision_worktree_data.py' to copy it from the main checkout",
)
# Resolve the serving release before initializing state-backed persistence. The optional
# seeders below depend on both boundaries: first_run_load reads the resolved serving corpus
# and writes compliance/watchlist state, while saved views require the initialized intel DB.
_initialize_required_persistence(app)
_run_optional_startup_seeders()
if distribution() is Distribution.FULL:
load_full_registration().preflight_cert_format_catalog()
try:
from sinetti_data_reads.curated_results import customer_mode as _customer_results_mode
from sinetti_services import lifecycle_source
if not lifecycle_source.history_available():
if _customer_results_mode():
# Expected on every customer install (owner decision 2026-10-22, option b):
# no refresh history ships yet, so this is information, not an operator fault.
obs.info(
"delisting_inference_unavailable",
path=str(state_paths.refresh_dir() / "change_events.jsonl"),
hint="customer install: no change history ships with this release, so "
"Ask's lifecycle_history tool cannot detect delisted certificates -- "
"every 'delisted: false' it reports means 'could not check', not "
"'confirmed clean'. Change history arrives with a later data release; "
"nothing to run on this host.",
)
else:
obs.error(
"delisting_inference_unavailable",
path=str(state_paths.refresh_dir() / "change_events.jsonl"),
hint="Ask's lifecycle_history tool cannot detect delisted certificates on "
"this install -- every 'delisted: false' it reports means 'could not "
"check', not 'confirmed clean'. Fresh git worktree: run "
"'python scripts/provision_worktree_data.py'. Fresh install: wait for "
"the first daily refresh to build change_events.jsonl.",
)
else:
obs.info("delisting_inference_ready")
except Exception as exc: # noqa: BLE001 β enrichment preflight, never block boot
obs.error(
"delisting_inference_preflight_failed",
error_type=type(exc).__name__,
error=str(exc),
)
# Data plane first (synchronous): download and validate the latest candidate before prewarm,
# but never overlay it onto serving data. Promotion is a reviewed deployment operation.
# It is env-gated and fail-open; this guard ensures startup still cannot die here.
if distribution() is Distribution.FULL:
load_full_registration().boot_sync()
# All selected warm work blocks readiness so customer requests cannot contend with CPU-heavy
# cache construction. SINETTI_PREWARM_OPTIONAL=0 keeps critical + landing only;
# SINETTI_PREWARM=0 (test suite) skips every phase. /api/ready still reports frontend.stale
# from frontend_build_status independently of these warm phases.
if os.environ.get("SINETTI_PREWARM", "1") != "0":
_run_prewarm_phase("critical", _prewarm_critical)
else:
obs.info("prewarm_phase_skipped", phase="critical", reason="SINETTI_PREWARM=0")
_prewarm_before_ready()
# After every warm phase, before the first request: the warmed heap is the boot heap.
_freeze_boot_heap()
if distribution() is Distribution.FULL:
stop_regulatory_scheduler = load_full_registration().start_regulatory_scheduler(app)
else:
stop_regulatory_scheduler = None
from sinetti_services.ask_cohort_store import start_retention_worker
stop_ask_retention = start_retention_worker()
heartbeat = asyncio.create_task(obs.worker_heartbeat(), name="worker-heartbeat")
try:
yield
finally:
heartbeat.cancel()
with suppress(asyncio.CancelledError):
await heartbeat
stop_ask_retention()
if distribution() is Distribution.FULL:
stop_regulatory_scheduler()
def _default_static_dir() -> Path | None:
from sinetti_services.frontend_build_status import served_dist_dir
dist = served_dist_dir(Path(__file__).parents[1])
return dist if dist.exists() else None
class _PersistenceSchemaError(RuntimeError):
pass
def _required_persistence_targets() -> dict[str, tuple[dict, ...]]:
import enrich_store
import state_paths
import store_billing
import store_iscc
from sinetti_services import ai_meter
from sinetti_services import data_access_meter
from sinetti_services import ops_error_ledger
from sinetti_services import ops_ownership
intel_required_schema = dict(_INTEL_REQUIRED_SCHEMA)
intel_required_indexes = set(_INTEL_REQUIRED_INDEXES)
private_dialect = store_iscc._private_data_persistence_config().dialect
if private_dialect == "postgres":
# Residual hub SQLite after Track C: live certificate cache only.
intel_required_schema = {
"iscc_certs": _INTEL_REQUIRED_SCHEMA["iscc_certs"],
}
intel_required_indexes = {"ix_iscc_holder", "ix_iscc_country"}
enrich_required_schema = dict(_ENRICH_REQUIRED_SCHEMA)
if enrich_store._history_persistence_config().dialect == "postgres":
# enrichment_cache + enrichment_history both live on the private plane.
enrich_required_schema = {}
targets = {
"intel": (
{
"path": Path(store_iscc.DB_PATH),
"tables": intel_required_schema,
"indexes": frozenset(intel_required_indexes),
},
{
"path": state_paths.intel_db_path(),
"tables": enrich_required_schema,
"indexes": frozenset(),
},
),
"ai_meter": (
{
"path": Path(ai_meter.DB_PATH),
"tables": _AI_REQUIRED_SCHEMA,
"indexes": _AI_REQUIRED_INDEXES,
},
),
"data_access_meter": (
{
"path": Path(data_access_meter.DB_PATH),
"tables": _DATA_ACCESS_REQUIRED_SCHEMA,
"indexes": _DATA_ACCESS_REQUIRED_INDEXES,
},
),
"ops_error_ledger": (
{
"path": Path(ops_error_ledger.DB_PATH),
"tables": _OPS_ERROR_REQUIRED_SCHEMA,
"indexes": _OPS_ERROR_REQUIRED_INDEXES,
},
),
}
if store_billing._private_data_persistence_config().dialect == "sqlite":
import store_auth
targets["auth"] = (
{
"path": Path(store_auth.DB_PATH),
"tables": _AUTH_REQUIRED_SCHEMA,
"indexes": _AUTH_REQUIRED_INDEXES,
},
)
targets["billing"] = (
{
"path": Path(store_billing.DB_PATH),
"tables": _BILLING_REQUIRED_SCHEMA,
"indexes": frozenset(),
},
)
targets["ops_ownership"] = (
{
"path": Path(ops_ownership.DB_PATH),
"tables": _OPS_OWNERSHIP_REQUIRED_SCHEMA,
"indexes": _OPS_OWNERSHIP_REQUIRED_INDEXES,
},
)
return targets
def _probe_sqlite_schema(
target: dict,
*,
busy_timeout_ms: int = PERSISTENCE_PROBE_LOCK_BUDGET_MS,
) -> None:
path = Path(target["path"]).resolve()
uri = f"{path.as_uri()}?mode=rw"
connection = sqlite3.connect(
uri,
uri=True,
timeout=busy_timeout_ms / 1000,
)
try:
connection.row_factory = sqlite3.Row
connection.execute(f"PRAGMA busy_timeout = {busy_timeout_ms}")
connection.execute("SELECT 1").fetchone()
for table, required_columns in target["tables"].items():
actual_columns = {
str(row["name"])
for row in connection.execute(
"SELECT name FROM pragma_table_info(?)",
(table,),
).fetchall()
}
missing_columns = set(required_columns) - actual_columns
if missing_columns:
raise _PersistenceSchemaError(
f"{table} is missing columns: {', '.join(sorted(missing_columns))}"
)
actual_indexes = {
str(row["name"])
for row in connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'index'"
).fetchall()
}
missing_indexes = set(target["indexes"]) - actual_indexes
if missing_indexes:
raise _PersistenceSchemaError(
f"missing indexes: {', '.join(sorted(missing_indexes))}"
)
connection.execute("BEGIN IMMEDIATE")
connection.execute(
"CREATE TABLE IF NOT EXISTS __sinetti_write_probe (probe INTEGER NOT NULL)"
)
connection.execute("INSERT INTO __sinetti_write_probe (probe) VALUES (1)")
connection.rollback()
finally:
if connection.in_transaction:
connection.rollback()
connection.close()
def _is_lock_contention(exc: BaseException) -> bool:
"""A busy/locked SQLite file β the app's own traffic β not a structural fault."""
if not isinstance(exc, sqlite3.OperationalError):
return False
message = str(exc).lower()
return "locked" in message or "busy" in message
def _probe_required_persistence(
targets: dict[str, tuple[dict, ...]],
contention_streaks: dict[str, int] | None = None,
) -> dict:
checks: dict[str, dict[str, object]] = {}
streaks = contention_streaks if contention_streaks is not None else {}
started = time.perf_counter()
for component, component_targets in targets.items():
try:
for target in component_targets:
spent_ms = (time.perf_counter() - started) * 1000
remaining_ms = PERSISTENCE_PROBE_LOCK_BUDGET_MS - spent_ms
_probe_sqlite_schema(
target,
busy_timeout_ms=int(max(PERSISTENCE_PROBE_MIN_TIMEOUT_MS, remaining_ms)),
)
except Exception as exc:
if _is_lock_contention(exc):
streak = streaks.get(component, 0) + 1
streaks[component] = streak
tolerated = streak <= PERSISTENCE_PROBE_CONTENTION_TOLERANCE
checks[component] = {
"ok": tolerated,
"required": True,
"reason": "contended",
}
report = obs.warn if tolerated else obs.error
report(
"required_persistence_probe_contended"
if tolerated
else "required_persistence_probe_failed",
component=component,
error_type=type(exc).__name__,
error=str(exc),
consecutive=streak,
tolerated=tolerated,
)
else:
streaks[component] = 0
reason = (
"schema incomplete"
if isinstance(exc, _PersistenceSchemaError)
else "unavailable"
)
checks[component] = {"ok": False, "required": True, "reason": reason}
obs.error(
"required_persistence_probe_failed",
component=component,
error_type=type(exc).__name__,
error=str(exc),
)
else:
streaks[component] = 0
checks[component] = {"ok": True, "required": True, "reason": ""}
return {
"ok": all(bool(check["ok"]) for check in checks.values()),
"checks": checks,
}
def _heal_missing_required_columns(targets: dict[str, tuple[dict, ...]]) -> None:
"""Add any required column the initializers and targeted migrations left missing.
The persistent bucket DBs carry years of drift; the 2026-08-01 deploy found shapes no
code generation ever created fresh (a `dismissals` without `username`), so boot could
finish green while the readiness probe stayed 'schema incomplete' forever. Healing is
strictly additive β `ALTER TABLE ... ADD COLUMN` (nullable), never a drop or rewrite β
and runs AFTER the targeted tenant migrations so it only patches residual drift.
Missing tables and indexes stay the initializers' job (they already CREATE IF NOT
EXISTS); a table absent here is left for the probe to report honestly.
"""
for component, component_targets in targets.items():
for target in component_targets:
path = Path(target["path"])
if not path.is_file():
continue
try:
connection = sqlite3.connect(path)
except Exception: # noqa: BLE001 - healing is best-effort; the probe reports truth
continue
try:
connection.row_factory = sqlite3.Row
for table, required_columns in target["tables"].items():
actual = {
str(row["name"])
for row in connection.execute(
"SELECT name FROM pragma_table_info(?)", (table,)
).fetchall()
}
if not actual:
continue
for column in sorted(set(required_columns) - actual):
try:
connection.execute(
f'ALTER TABLE "{table}" ADD COLUMN "{column}" TEXT'
)
obs.info(
"required_schema_healed",
component=component,
table=table,
column=column,
)
except Exception as exc: # noqa: BLE001 - report, keep healing the rest
obs.error(
"required_schema_heal_failed",
component=component,
table=table,
column=column,
error=str(exc),
)
connection.commit()
finally:
connection.close()
def _initialize_required_persistence(app: FastAPI) -> None:
checks: dict[str, dict[str, object]] = {}
def run(component: str, initializer) -> None:
try:
initializer()
except Exception as exc:
checks[component] = {
"ok": False,
"required": True,
"reason": "initialization failed",
}
obs.error(
"required_persistence_initialization_failed",
component=component,
error_type=type(exc).__name__,
error=str(exc),
)
else:
checks[component] = {"ok": True, "required": True, "reason": ""}
def initialize_intel() -> None:
import store_iscc
store_iscc.init_iscc_tables()
def initialize_ai_meter() -> None:
from sinetti_services import ai_meter
ai_meter.connect().close()
def initialize_data_access_meter() -> None:
from sinetti_services import data_access_meter
data_access_meter.connect().close()
def initialize_ops_error_ledger() -> None:
from sinetti_services import ops_error_ledger
ops_error_ledger.connect().close()
def initialize_ops_ownership() -> None:
from sinetti_services import ops_ownership
ops_ownership.connect().close()
def initialize_billing() -> None:
import store_billing
store_billing.init_billing_tables()
store_billing.seed_billing_defaults()
def initialize_auth() -> None:
import store_auth
store_auth.init_auth_tables()
# Billing membership must exist before intel tenant migration stamps org_id.
run("billing", initialize_billing)
run("auth", initialize_auth)
run("intel", initialize_intel)
run("ai_meter", initialize_ai_meter)
run("data_access_meter", initialize_data_access_meter)
run("ops_error_ledger", initialize_ops_error_ledger)
run("ops_ownership", initialize_ops_ownership)
startup = {
"ok": all(bool(check["ok"]) for check in checks.values()),
"checks": checks,
}
targets = _required_persistence_targets()
try:
_heal_missing_required_columns(targets)
except Exception as exc: # noqa: BLE001 - healing must never take the app down
obs.error(
"required_schema_heal_failed",
component="*",
error_type=type(exc).__name__,
error=str(exc),
)
app.state.required_persistence_startup = startup
app.state.required_persistence = startup
# Per-app, per-worker: how many consecutive probes each component has been locked for.
contention_streaks: dict[str, int] = {}
app.state.required_persistence_probe = lambda: _probe_required_persistence(
targets, contention_streaks
)
def _run_optional_startup_seeders() -> None:
"""Best-effort startup hydration after release resolution and table initialization."""
import iscc
import store_iscc
seeders = (
("saved_views_seed", store_iscc.import_saved_views_seed),
("iscc_compliance_seed", iscc.first_run_load),
)
for name, seeder in seeders:
try:
result = seeder()
except Exception as exc: # noqa: BLE001 β optional hydration must never block V2 boot
obs.error(
"optional_startup_seed_failed",
seed=name,
error_type=type(exc).__name__,
error=str(exc),
)
else:
details = {"imported": result} if isinstance(result, int) else {}
obs.info("optional_startup_seed_complete", seed=name, **details)
def _customer_router(router: APIRouter) -> APIRouter:
"""The router the active edition registers: as is in the full edition, else a copy without
the trimmed operations (route_access.CUSTOMER_TRIMMED_OPERATIONS), such as the NEa line of
the holder router. FastAPI includes routers by reference, so the shared module router itself
is never altered and a full app built in the same process keeps every route.
"""
if distribution() is Distribution.FULL:
return router
from fastapi.routing import APIRoute
from sinetti_api.route_access import CUSTOMER_TRIMMED_OPERATIONS
if router.dependencies or router.on_startup or router.on_shutdown:
raise RuntimeError("customer route trim copies plain routers only")
kept = APIRouter()
kept.routes.extend(
route
for route in router.routes
if not (
isinstance(route, APIRoute)
and any((method, route.path) in CUSTOMER_TRIMMED_OPERATIONS for method in route.methods)
)
)
return kept
def create_app(static_dir: str | Path | None = None) -> FastAPI:
# The fixed full composition requires its private add-on before mounting any route.
if distribution() is Distribution.FULL:
operator = load_full_registration()
else:
operator = None
from sinetti_api.runtime_security import hosted_runtime, local_loopback_request
from sinetti_api.browser_session import BrowserSessionMiddleware
hosted = hosted_runtime()
app = FastAPI(
title="Sinetti API", version="2.0.0", lifespan=_lifespan,
docs_url=None if hosted else "/docs",
redoc_url=None if hosted else "/redoc",
openapi_url=None if hosted else "/openapi.json",
)
# Compress responses: the certificate/holder/analytics JSON is large and highly repetitive
# (identical keys per row), so gzip shrinks it ~90%. Biggest single network win for Browse.
app.add_middleware(GZipMiddleware, minimum_size=1000)
app.add_middleware(BatchCheckUploadLimitMiddleware)
app.add_middleware(BrowserSessionMiddleware)
@app.middleware("http")
async def request_observability(request, call_next):
# A caller-supplied X-Request-ID is kept only when it looks like an id; anything else
# would put arbitrary text into logs and the error ledger (diagnostics_contract).
trace_id = _diagnostics.safe_trace_id(request.headers.get("X-Request-ID"))
request.state.trace_id = trace_id
started = time.perf_counter()
if request.url.path in {"/docs", "/docs/oauth2-redirect", "/redoc", "/openapi.json"} and not local_loopback_request(request):
response = JSONResponse(status_code=404, content={"ok": False, "message": "Not found"})
else:
response = await call_next(request)
response.headers["X-Request-ID"] = trace_id
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["X-Frame-Options"] = "DENY"
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()"
if hosted_runtime() or request.url.scheme == "https":
response.headers["Strict-Transport-Security"] = "max-age=31536000"
response.headers["Content-Security-Policy"] = (
"default-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; "
"script-src 'self'; style-src 'self' 'unsafe-inline'; "
# MapLibre GL spawns its render worker from a blob: URL. Without this the
# worker throws a SecurityError and the Browse error boundary swallows the
# whole page. Scoped to worker-src so script-src stays 'self' only.
"worker-src 'self' blob:; "
# The intro video is fetched with the session's Bearer header (a <video src> cannot
# send one) and played from a blob: URL. Media only: script-src is untouched.
"media-src 'self' blob:; "
# img-src https: covers raster tile pixels; connect-src must also allow
# MapTiler TileJSON/tile fetches or Satellite fails closed under CSP.
"img-src 'self' data: https:; connect-src 'self' https://api.maptiler.com"
)
duration_ms = (time.perf_counter() - started) * 1000
memory = {"rss_mb": obs.worker_rss_mb()} if duration_ms > 2000 else {}
obs.info(
"request_complete",
pid=os.getpid(),
trace_id=trace_id,
method=request.method,
path=_diagnostics.log_path(request),
status=response.status_code,
duration_ms=round(duration_ms, 1),
**memory,
)
return response
app.add_exception_handler(RequestValidationError, request_validation_exception_handler)
app.add_exception_handler(StarletteHTTPException, http_exception_handler)
app.add_exception_handler(FeatureBlocked, feature_blocked_handler)
app.add_exception_handler(ActionUnavailable, action_unavailable_handler)
app.add_exception_handler(TenantRequired, tenant_required_handler)
app.add_exception_handler(Exception, unhandled_exception_handler)
# Open routes: health (liveness) + auth (login/session/logout), plus email_public's
# unsubscribe and copy routes (token-authenticated, not session-authenticated by design:
# both are opened from a mail, where there is no session to carry). Every
# other router carries data and requires a verified session (require_user). setup/triage
# additionally require the admin flag. No anonymous data access.
# research_scope: per-plan research visibility (owner 2026-09-29) for every user-facing route.
_user = [Depends(require_user), Depends(research_scope)]
_data = _user + [Depends(require_data_access)]
app.include_router(health.router)
app.include_router(auth.router)
app.include_router(security.router, dependencies=_user)
# Customer edition registration trim (owner 2026-10-22, SIN-0896): Ask, Counterparty Profile,
# Business Footprint and operator monitoring are registered by the full edition only; the
# operation list is route_access.CUSTOMER_TRIMMED_OPERATIONS. Nothing is deleted.
if distribution() is Distribution.FULL:
app.include_router(chat.router, dependencies=_user)
app.include_router(ask_conversations.router, dependencies=_user)
app.include_router(discussions.router, dependencies=_user)
app.include_router(capabilities.router, dependencies=_user)
app.include_router(fuel_conversion.router, dependencies=_user)
app.include_router(fuel_specs.router, dependencies=_user)
app.include_router(intro_video.router, dependencies=_user)
# Gazetteer lookup only β no certificate data, so it rides _user rather than _data.
app.include_router(places.router, dependencies=_user)
app.include_router(contact.router, dependencies=_user)
app.include_router(feedback.router, dependencies=_user)
app.include_router(certificates.router, dependencies=_data)
if distribution() is Distribution.FULL:
app.include_router(operator.certificates_admin, dependencies=[Depends(require_admin)])
app.include_router(_customer_router(holders.router), dependencies=_data)
if distribution() is Distribution.FULL:
app.include_router(operator.holders_admin, dependencies=[Depends(require_admin)])
app.include_router(site_footprint.router, dependencies=_data)
app.include_router(site_assessment.router, dependencies=_data)
app.include_router(
dossier.router,
dependencies=_data + [Depends(require_feature("business_footprint"))],
)
app.include_router(risk.router, dependencies=_data)
app.include_router(saf_exports.router, dependencies=_data)
app.include_router(saf_exports.index_router, dependencies=_user)
app.include_router(activity.router, dependencies=_user)
app.include_router(saved_views.router, dependencies=_user)
if distribution() is Distribution.FULL:
app.include_router(monitor.router, dependencies=_data)
app.include_router(feedstocks.router, dependencies=_data)
app.include_router(fuels.router, dependencies=_data)
app.include_router(geography.router, dependencies=_data)
if distribution() is Distribution.FULL:
app.include_router(certification_bodies.router, dependencies=_data)
app.include_router(category_timelines.router, dependencies=_data)
app.include_router(maps.router, dependencies=_data)
if distribution() is Distribution.FULL:
app.include_router(operator.maps_admin, dependencies=[Depends(require_admin)])
# Resolved family membership drives the Browse filter, which every entitled user sees β so
# this is _data, NOT admin. The admin-only editing surface stays under /api/setup.
app.include_router(material_families.router, dependencies=_data)
app.include_router(search.router, dependencies=_data)
# Counterparty DD / Business Footprint are billing-gated features (whole router β every
# route in each is the gated surface): require_feature raises FeatureBlocked -> HTTP 200
# graceful envelope, never a bare 403 (see sinetti_api/errors.py).
if distribution() is Distribution.FULL:
app.include_router(
counterparty.router,
dependencies=_data + [Depends(require_feature("counterparty_dd"))],
)
# Counterparty Fit (Task 6): a separate admin-only capability, NOT gated by the counterparty_dd
# billing feature -- an admin whose org lacks that paid feature must still be able to use this
# internal diagnostic tool. Same paired router/admin_router split as certificates/holders/maps
# below: only require_admin, no _data, no require_feature.
if distribution() is Distribution.FULL:
app.include_router(operator.counterparty_admin, dependencies=[Depends(require_admin)])
if distribution() is Distribution.FULL:
app.include_router(
business_footprint.router,
dependencies=_user + [
Depends(require_feature("business_footprint")),
Depends(require_research_data),
],
)
app.include_router(timeline.router, dependencies=_user)
app.include_router(tours.router, dependencies=_user)
app.include_router(watchlist.router, dependencies=_user)
app.include_router(soft_presets.router, dependencies=_user)
app.include_router(notifications.router, dependencies=_user)
app.include_router(me_profile.router, dependencies=_user)
app.include_router(ops.router, dependencies=_user)
if distribution() is Distribution.FULL:
app.include_router(operator.ops_admin, dependencies=[Depends(require_admin)])
else:
app.include_router(ops.customer_router, dependencies=_user)
app.include_router(product_signals.router, dependencies=_user)
app.include_router(email_public.router)
app.include_router(access_public.router)
if distribution() is Distribution.FULL:
app.include_router(operator.access_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.email_admin, dependencies=_user)
else:
from sinetti_api.routers import access_admin as customer_access_admin
from sinetti_api.routers import customer_email_delivery
app.include_router(customer_access_admin.customer_router, dependencies=_user)
app.include_router(customer_access_admin.settings_router, dependencies=_user)
app.include_router(customer_email_delivery.router, dependencies=_user)
app.include_router(analytics.router, dependencies=_data)
app.include_router(archive_life.router, dependencies=_data)
if distribution() is Distribution.FULL:
app.include_router(sentinel.router, dependencies=_user)
app.include_router(operator.setup, dependencies=[Depends(require_admin)])
app.include_router(operator.cert_formats_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.knowledge_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.data_trust_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.manual_refresh_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.research_coverage_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.regulatory_monitor_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.triage, dependencies=[Depends(require_admin)])
app.include_router(operator.billing_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.customer_data_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.data_access_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.admin_sources, dependencies=[Depends(require_admin)])
app.include_router(operator.ai_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.ask_abuse_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.feedback_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.product_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.sales_pitch_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.sales_deck_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.admin_guide_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.admin_legal_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.site_footprint_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.site_assessment_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.soft_presets_admin, dependencies=[Depends(require_admin)])
app.include_router(operator.tours_admin, dependencies=[Depends(require_admin)])
app.include_router(narrative.router, dependencies=_user)
app.include_router(flags.router, dependencies=_user)
market_balances.register(
app,
dependencies=_user + [Depends(require_feature("market_balances"))],
)
batch_check.register(
app,
dependencies=_data + [Depends(require_feature("batch_check"))],
)
app.include_router(about.router, dependencies=_user)
mount_dir = static_dir if static_dir is not None else _default_static_dir()
if mount_dir:
static_path = Path(mount_dir)
assets_path = static_path / "assets"
if assets_path.exists():
app.mount("/assets", StaticFiles(directory=assets_path), name="assets")
# All methods, not just GET: an unmatched api/ path must come back as a genuine 404
# regardless of the request's HTTP method. A GET-only catch-all would 405 a POST to an
# unmatched api/ path before this handler's own api/-aware 404 branch ever runs β
# Starlette rejects on method mismatch at the routing layer, never reaching the handler body.
# include_in_schema=False: the SPA catch-all is not part of the API contract, and a
# multi-method route reuses one generated operation ID per method, so documenting it
# emits FastAPI's duplicate-operation-ID warning on every openapi() build.
@app.api_route(
"/{full_path:path}",
methods=["GET", "POST", "PUT", "PATCH", "DELETE"],
response_class=HTMLResponse,
include_in_schema=False,
)
def react_fallback(full_path: str, request: Request) -> Response:
if full_path.startswith("api/"):
raise HTTPException(status_code=404, detail="Not found")
# Real files sitting in the built frontend ROOT must be served as themselves before the
# SPA shell below swallows them. Vite copies everything in frontend/public/ to the dist
# root, and only /assets (the hashed bundle output) is mounted above β so the MapLibre
# basemap (/basemap-z0-6.pmtiles) and the self-hosted glyph fonts (/fonts/**) were both
# answering with index.html, and the map rendered with no basemap and no labels while
# every health check stayed green (2026-07-25: found live on port 8511; the Playwright
# e2e suite never caught it because it runs against the Vite dev server, which serves
# public/ natively β only the FastAPI-served production build has this gap).
# FileResponse (Starlette >= 1.x) honours HTTP Range requests, which pmtiles REQUIRES:
# it reads the 45 MB archive in small byte ranges rather than downloading it whole.
if full_path and request.method in ("GET", "HEAD"):
static_root = static_path.resolve()
candidate = (static_root / full_path).resolve()
# Containment check first: a crafted path (../../secrets) must never escape dist.
if candidate.is_relative_to(static_root) and candidate.is_file():
return FileResponse(candidate)
# The React shell must never be cached: a stale/stub index.html gets served from cache
# and renders a blank white page. Hashed assets under /assets stay immutable-cached; only
# this entry document is marked no-store so the app always boots the current bundle.
html = (static_path / "index.html").read_text(encoding="utf-8")
return HTMLResponse(content=html, headers={"Cache-Control": "no-store"})
return app
app = create_app()
from __future__ import annotations
import os
import asyncio
import gc
import sqlite3
import sys
import time
import uuid
from collections.abc import Callable
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
from sinetti_api.distribution import Distribution, distribution, load_full_registration
if distribution() is Distribution.FULL:
import secrets_local
from sinetti_services import serving_recovery
import obs
import state_paths
from sinetti_api.addons import batch_check, market_balances
from sinetti_api.upload_limits import BatchCheckUploadLimitMiddleware
if distribution() is Distribution.FULL:
# Mirror Classic (app.py): load AI/search keys from a local .env for APP_BACKEND=local runs.
# Gated + no-ops on HF (existing env / Space Secrets win). Without this, V2 uvicorn starts
# with a bare environment and Sentinel/chat see a missing ANTHROPIC_API_KEY even though .env
# has it. The fixed CUSTOMER composition does not ship the desktop dotenv helper.
secrets_local.load_local_dotenv()
# Private mode (Private Cloud): arm the outbound-connection guard before anything - boot data
# sync, prewarm, any router - can open a socket. No-op unless SINETTI_PRIVATE_MODE=1.
from sinetti_services import egress_guard as _egress_guard
_egress_guard.install()
from sinetti_services import diagnostics_contract as _diagnostics # noqa: E402
from sinetti_api.deps import (
require_admin,
require_data_access,
require_feature,
require_research_data,
research_scope,
require_user,
)
from sinetti_api.errors import (
ActionUnavailable,
FeatureBlocked,
action_unavailable_handler,
feature_blocked_handler,
http_exception_handler,
request_validation_exception_handler,
tenant_required_handler,
unhandled_exception_handler,
)
from sinetti_services.tenant_context import TenantRequired
from sinetti_api.routers import (
about,
access_public,
ask_conversations,
discussions,
analytics,
archive_life,
auth,
business_footprint,
capabilities,
category_timelines,
certificates,
certification_bodies,
chat,
contact,
feedback,
counterparty,
dossier,
email_public,
feedstocks,
flags,
fuel_conversion,
fuel_specs,
intro_video,
fuels,
geography,
health,
holders,
maps,
material_families,
me_profile,
monitor,
narrative,
notifications,