Spaces:
Running
Running
Sync DemoService e2b09b366a64790f3dd1dc93870f72061e3c1aab
Browse files- Dockerfile +1 -0
- api/ai-service/app/api/v1/endpoints/route_generation.py +25 -1
- api/ai-service/app/api/v1/endpoints/route_library.py +177 -193
- api/ai-service/app/schemas/__init__.py +1 -0
- api/ai-service/app/schemas/route_library.py +85 -0
- api/ai-service/app/services/art_route_catalog.py +20 -13
- api/ai-service/app/services/art_route_ingest.py +196 -0
- api/ai-service/app/services/route_library_store.py +289 -0
- api/ai-service/deployment-source.json +1 -0
- api/ai-service/tests/test_route_library_deployment.py +53 -0
- api/ai-service/tests/test_route_library_postgres.py +613 -0
- api/auth-service/alembic/env.py +20 -8
- api/auth-service/alembic/versions/0014_link_lab_art_routes.py +94 -0
- api/auth-service/requirements.txt +1 -1
- deploy/huggingface/route-library-preflight.py +82 -0
- deploy/huggingface/start-hf.sh +2 -5
- docs/LAB_GPX_SYNC.md +135 -0
- docs/LOCAL_ART_ROUTE_CATALOG.md +24 -0
- tools/lab_sync/README.md +21 -0
- tools/lab_sync/sync_lab_gpx.py +767 -0
- tools/lab_sync/tests/fixtures/approval_guards.sql +129 -0
- tools/lab_sync/tests/fixtures/approval_policy.py +54 -0
- tools/lab_sync/tests/test_lab_sender_v2_offline.py +617 -0
- tools/sync_lab_gpx.py +294 -10
Dockerfile
CHANGED
|
@@ -79,6 +79,7 @@ COPY api /app/api
|
|
| 79 |
COPY router /app/router
|
| 80 |
COPY --from=frontend-builder /build/main /app/main
|
| 81 |
COPY deploy/huggingface/start-hf.sh /app/start-hf.sh
|
|
|
|
| 82 |
COPY deploy/huggingface/running-environment-profiles.tar.gz.enc /app/running-environment-profiles.tar.gz.enc
|
| 83 |
|
| 84 |
RUN chmod +x /app/start-hf.sh \
|
|
|
|
| 79 |
COPY router /app/router
|
| 80 |
COPY --from=frontend-builder /build/main /app/main
|
| 81 |
COPY deploy/huggingface/start-hf.sh /app/start-hf.sh
|
| 82 |
+
COPY deploy/huggingface/route-library-preflight.py /app/route-library-preflight.py
|
| 83 |
COPY deploy/huggingface/running-environment-profiles.tar.gz.enc /app/running-environment-profiles.tar.gz.enc
|
| 84 |
|
| 85 |
RUN chmod +x /app/start-hf.sh \
|
api/ai-service/app/api/v1/endpoints/route_generation.py
CHANGED
|
@@ -4,6 +4,7 @@ import importlib.util
|
|
| 4 |
import logging
|
| 5 |
import sys
|
| 6 |
import time
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
from types import ModuleType
|
| 9 |
import math
|
|
@@ -19,7 +20,7 @@ from app.services.gpx_shape_similarity import (
|
|
| 19 |
)
|
| 20 |
from app.services.walking_route_fallback import generate_walking_fallback
|
| 21 |
from app.services.route_display_orientation import attach_card_display_orientations, attach_display_orientations
|
| 22 |
-
from app.api.v1.endpoints.route_library import active_outlines
|
| 23 |
|
| 24 |
router = APIRouter(prefix="/ai", tags=["ai"])
|
| 25 |
SHAPE_DISCOVERY_LABELS = ("dog", "symbol_star", "t_shirt", "heart", "candle", "christmas_tree_trunk")
|
|
@@ -30,6 +31,17 @@ logger = logging.getLogger("uvicorn.error")
|
|
| 30 |
_PIPELINE_MODULE_CACHE: dict[str, tuple[int, ModuleType]] = {}
|
| 31 |
|
| 32 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
class RouteGenerateRequest(BaseModel):
|
| 34 |
source: Literal["place", "current"] = "place"
|
| 35 |
place: str | None = None
|
|
@@ -396,6 +408,13 @@ def _run_fast_road_pipeline(
|
|
| 396 |
catalog_outline_ids = {id(outline) for outline in catalog_entry[1]} if catalog_entry else set()
|
| 397 |
if examples_override is not None:
|
| 398 |
examples = examples_override
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 399 |
candidate_pool = min(16 if examples else 8, max(route_count, route_count + (11 if examples else 1)))
|
| 400 |
if (shape_priority or catalog_entry or examples_override is not None) and examples:
|
| 401 |
# Explore multiple physical sizes on the same road graph. The UI did
|
|
@@ -579,6 +598,7 @@ def _run_fast_road_pipeline(
|
|
| 579 |
|
| 580 |
|
| 581 |
@router.post("/routes/discover-shapes")
|
|
|
|
| 582 |
def discover_shapes(payload: ShapeDiscoveryRequest) -> dict[str, Any]:
|
| 583 |
"""Find runnable shape outlines near a picked map point."""
|
| 584 |
workspace_dir = Path(settings.ai_workspace_dir)
|
|
@@ -809,6 +829,7 @@ def _coordinate_fallback_routes(
|
|
| 809 |
|
| 810 |
|
| 811 |
@router.post("/routes/generate")
|
|
|
|
| 812 |
def generate_routes(payload: RouteGenerateRequest) -> dict[str, Any]:
|
| 813 |
started_at = time.perf_counter()
|
| 814 |
logger.info("route_generation request start automatic=%s shape_priority=%s route_count=%s",
|
|
@@ -840,6 +861,9 @@ AUTO_MAX_LABELS = 4
|
|
| 840 |
|
| 841 |
def _generate_automatic_shapes(module, payload, labels, catalog, radius_km, route_count):
|
| 842 |
"""Compare nearby road fits across labels; never fill with generic loops."""
|
|
|
|
|
|
|
|
|
|
| 843 |
started_at = time.perf_counter()
|
| 844 |
timings_ms: dict[str, int] = {}
|
| 845 |
build_graph = module._build_or_get_walk_graph
|
|
|
|
| 4 |
import logging
|
| 5 |
import sys
|
| 6 |
import time
|
| 7 |
+
from functools import wraps
|
| 8 |
from pathlib import Path
|
| 9 |
from types import ModuleType
|
| 10 |
import math
|
|
|
|
| 20 |
)
|
| 21 |
from app.services.walking_route_fallback import generate_walking_fallback
|
| 22 |
from app.services.route_display_orientation import attach_card_display_orientations, attach_display_orientations
|
| 23 |
+
from app.api.v1.endpoints.route_library import active_outlines, catalog_revision
|
| 24 |
|
| 25 |
router = APIRouter(prefix="/ai", tags=["ai"])
|
| 26 |
SHAPE_DISCOVERY_LABELS = ("dog", "symbol_star", "t_shirt", "heart", "candle", "christmas_tree_trunk")
|
|
|
|
| 31 |
_PIPELINE_MODULE_CACHE: dict[str, tuple[int, ModuleType]] = {}
|
| 32 |
|
| 33 |
|
| 34 |
+
def _consistent_catalog(function):
|
| 35 |
+
@wraps(function)
|
| 36 |
+
def guarded(*args, **kwargs):
|
| 37 |
+
revision = catalog_revision()
|
| 38 |
+
result = function(*args, **kwargs)
|
| 39 |
+
if revision is not None and catalog_revision() != revision:
|
| 40 |
+
raise HTTPException(409, detail="route_library_changed_retry")
|
| 41 |
+
return result
|
| 42 |
+
return guarded
|
| 43 |
+
|
| 44 |
+
|
| 45 |
class RouteGenerateRequest(BaseModel):
|
| 46 |
source: Literal["place", "current"] = "place"
|
| 47 |
place: str | None = None
|
|
|
|
| 408 |
catalog_outline_ids = {id(outline) for outline in catalog_entry[1]} if catalog_entry else set()
|
| 409 |
if examples_override is not None:
|
| 410 |
examples = examples_override
|
| 411 |
+
seen_outlines, unique_examples = set(), []
|
| 412 |
+
for outline in examples:
|
| 413 |
+
signature = tuple(map(tuple, outline))
|
| 414 |
+
if signature not in seen_outlines:
|
| 415 |
+
seen_outlines.add(signature)
|
| 416 |
+
unique_examples.append(outline)
|
| 417 |
+
examples = unique_examples
|
| 418 |
candidate_pool = min(16 if examples else 8, max(route_count, route_count + (11 if examples else 1)))
|
| 419 |
if (shape_priority or catalog_entry or examples_override is not None) and examples:
|
| 420 |
# Explore multiple physical sizes on the same road graph. The UI did
|
|
|
|
| 598 |
|
| 599 |
|
| 600 |
@router.post("/routes/discover-shapes")
|
| 601 |
+
@_consistent_catalog
|
| 602 |
def discover_shapes(payload: ShapeDiscoveryRequest) -> dict[str, Any]:
|
| 603 |
"""Find runnable shape outlines near a picked map point."""
|
| 604 |
workspace_dir = Path(settings.ai_workspace_dir)
|
|
|
|
| 829 |
|
| 830 |
|
| 831 |
@router.post("/routes/generate")
|
| 832 |
+
@_consistent_catalog
|
| 833 |
def generate_routes(payload: RouteGenerateRequest) -> dict[str, Any]:
|
| 834 |
started_at = time.perf_counter()
|
| 835 |
logger.info("route_generation request start automatic=%s shape_priority=%s route_count=%s",
|
|
|
|
| 861 |
|
| 862 |
def _generate_automatic_shapes(module, payload, labels, catalog, radius_km, route_count):
|
| 863 |
"""Compare nearby road fits across labels; never fill with generic loops."""
|
| 864 |
+
refresh_templates = getattr(module, "apply_imported_shape_templates", None)
|
| 865 |
+
if callable(refresh_templates):
|
| 866 |
+
refresh_templates(active_outlines())
|
| 867 |
started_at = time.perf_counter()
|
| 868 |
timings_ms: dict[str, int] = {}
|
| 869 |
build_graph = module._build_or_get_walk_graph
|
api/ai-service/app/api/v1/endpoints/route_library.py
CHANGED
|
@@ -1,229 +1,213 @@
|
|
| 1 |
-
"""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
-
import base64
|
| 5 |
-
import binascii
|
| 6 |
-
import hashlib
|
| 7 |
import hmac
|
| 8 |
-
import io
|
| 9 |
import json
|
| 10 |
-
import math
|
| 11 |
import os
|
| 12 |
-
import re
|
| 13 |
-
import sys
|
| 14 |
-
import time
|
| 15 |
-
import xml.etree.ElementTree as ET
|
| 16 |
from pathlib import Path
|
| 17 |
from threading import Lock
|
|
|
|
| 18 |
|
| 19 |
import psycopg
|
| 20 |
-
from psycopg.
|
| 21 |
-
from fastapi import APIRouter, Header, HTTPException,
|
| 22 |
-
from
|
|
|
|
| 23 |
|
| 24 |
from app.core.config import settings
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
_cache_lock = Lock()
|
| 37 |
-
|
| 38 |
-
_cache
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
class RouteImport(BaseModel):
|
| 42 |
-
label: str
|
| 43 |
-
region_code: str = Field(max_length=100)
|
| 44 |
-
region_name: str = Field(max_length=200)
|
| 45 |
-
sha256: str
|
| 46 |
-
accepted_at: str
|
| 47 |
-
review_status: str
|
| 48 |
-
ai_protocol: str = Field(max_length=200)
|
| 49 |
-
geometry_ok: bool
|
| 50 |
-
seoul_ok: bool
|
| 51 |
-
collection_finalized: bool = False
|
| 52 |
-
gpx_base64: str
|
| 53 |
-
|
| 54 |
|
| 55 |
-
class ApprovalSnapshot(BaseModel):
|
| 56 |
-
ids: list[str] = Field(max_length=10000)
|
| 57 |
-
count: int = Field(ge=0, le=10000)
|
| 58 |
-
ids_sha256: str
|
| 59 |
-
collection_finalized: bool = False
|
| 60 |
|
| 61 |
-
|
| 62 |
-
def _database_url() -> str:
|
| 63 |
url = os.environ.get("ROUTE_LIBRARY_DATABASE_URL", "")
|
| 64 |
if not url:
|
| 65 |
-
raise HTTPException(
|
| 66 |
return url.replace("postgresql+psycopg://", "postgresql://", 1)
|
| 67 |
|
| 68 |
|
| 69 |
-
def _authorize(authorization
|
| 70 |
expected = os.environ.get("ROUTE_LIBRARY_TOKEN", "")
|
| 71 |
supplied = authorization.removeprefix("Bearer ") if authorization else ""
|
| 72 |
if not expected:
|
| 73 |
-
raise HTTPException(
|
| 74 |
-
if not hmac.compare_digest(expected, supplied):
|
| 75 |
-
raise HTTPException(
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
def
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
""")
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def _outline(gpx: bytes, source_id: str) -> list[list[float]]:
|
| 100 |
try:
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
points = [(float(node.attrib["lat"]), float(node.attrib["lon"]))
|
| 109 |
-
for node in segments[0] if node.tag.rsplit("}", 1)[-1] == "trkpt"]
|
| 110 |
-
if len(points) < 12 or len(points) > 10000 or any(
|
| 111 |
-
not math.isfinite(lat) or not math.isfinite(lon) or abs(lat) > 90 or abs(lon) > 180
|
| 112 |
-
for lat, lon in points
|
| 113 |
-
):
|
| 114 |
-
raise ValueError("Invalid GPX coordinates")
|
| 115 |
-
if math.dist(points[0], points[-1]) > 1e-5:
|
| 116 |
-
raise ValueError("Expected a closed route")
|
| 117 |
-
script_dir = Path(settings.ai_workspace_dir) / "scripts"
|
| 118 |
-
if str(script_dir) not in sys.path:
|
| 119 |
-
sys.path.insert(0, str(script_dir))
|
| 120 |
-
from build_gpx_shape_templates import normalized_outline_from_track
|
| 121 |
-
return normalized_outline_from_track(points, source_id)
|
| 122 |
-
except (ET.ParseError, KeyError, ValueError, RecursionError) as exc:
|
| 123 |
-
raise HTTPException(status_code=422, detail=f"Invalid GPX: {exc}") from exc
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
def _invalidate_cache() -> None:
|
| 127 |
-
global _cache_until
|
| 128 |
with _cache_lock:
|
| 129 |
-
|
| 130 |
|
| 131 |
|
| 132 |
@router.put("/routes/{source_id}")
|
| 133 |
-
def put_route(source_id: str, payload: RouteImport
|
|
|
|
| 134 |
_authorize(authorization)
|
| 135 |
-
if not ID_PATTERN.fullmatch(source_id) or payload.label not in LABELS:
|
| 136 |
-
raise HTTPException(status_code=422, detail="Unknown route ID or label")
|
| 137 |
-
if payload.review_status != "ai_pass" or not payload.geometry_ok or not payload.seoul_ok:
|
| 138 |
-
raise HTTPException(status_code=422, detail="Route is not in an approved state")
|
| 139 |
-
try:
|
| 140 |
-
gpx = base64.b64decode(payload.gpx_base64, validate=True)
|
| 141 |
-
except (ValueError, binascii.Error) as exc:
|
| 142 |
-
raise HTTPException(status_code=422, detail="Invalid GPX base64") from exc
|
| 143 |
-
if not gpx or len(gpx) > MAX_GPX_BYTES:
|
| 144 |
-
raise HTTPException(status_code=413, detail="GPX size is outside the accepted range")
|
| 145 |
-
digest = hashlib.sha256(gpx).hexdigest()
|
| 146 |
-
if not hmac.compare_digest(digest, payload.sha256.lower()):
|
| 147 |
-
raise HTTPException(status_code=422, detail="GPX SHA-256 mismatch")
|
| 148 |
try:
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
_ensure_table(connection)
|
| 158 |
-
connection.execute("""
|
| 159 |
-
INSERT INTO lab_route_library
|
| 160 |
-
(source, source_id, label, region_code, region_name, gpx_sha256,
|
| 161 |
-
accepted_at, ai_protocol, collection_finalized, gpx, outline, active)
|
| 162 |
-
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,true)
|
| 163 |
-
ON CONFLICT (source, source_id) DO UPDATE SET
|
| 164 |
-
label=excluded.label, region_code=excluded.region_code,
|
| 165 |
-
region_name=excluded.region_name, gpx_sha256=excluded.gpx_sha256,
|
| 166 |
-
accepted_at=excluded.accepted_at, ai_protocol=excluded.ai_protocol,
|
| 167 |
-
collection_finalized=excluded.collection_finalized, gpx=excluded.gpx,
|
| 168 |
-
outline=excluded.outline, active=true, updated_at=now()
|
| 169 |
-
""", (SOURCE, source_id, LABELS[payload.label], payload.region_code,
|
| 170 |
-
payload.region_name, digest, accepted_at, payload.ai_protocol,
|
| 171 |
-
payload.collection_finalized, gpx, Jsonb(outline)))
|
| 172 |
-
_invalidate_cache()
|
| 173 |
-
return {"source_id": source_id, "label": LABELS[payload.label], "sha256": digest, "status": "stored"}
|
| 174 |
|
| 175 |
|
| 176 |
@router.post("/approval-snapshot")
|
| 177 |
-
def apply_snapshot(payload: ApprovalSnapshot
|
|
|
|
| 178 |
_authorize(authorization)
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
raise HTTPException(
|
| 183 |
-
|
| 184 |
-
if not hmac.compare_digest(digest, payload.ids_sha256.lower()):
|
| 185 |
-
raise HTTPException(status_code=422, detail="Approval list SHA-256 mismatch")
|
| 186 |
-
with psycopg.connect(_database_url()) as connection:
|
| 187 |
-
_ensure_table(connection)
|
| 188 |
-
with connection.transaction():
|
| 189 |
-
current = {row[0] for row in connection.execute(
|
| 190 |
-
"SELECT source_id FROM lab_route_library WHERE source=%s AND active=true", (SOURCE,))}
|
| 191 |
-
missing = set(payload.ids) - current
|
| 192 |
-
if missing:
|
| 193 |
-
raise HTTPException(status_code=409, detail={"missing_routes": len(missing)})
|
| 194 |
-
revoked = current - set(payload.ids)
|
| 195 |
-
if len(revoked) > max(20, len(current) // 20):
|
| 196 |
-
raise HTTPException(status_code=409, detail={"unexpected_deactivations": len(revoked)})
|
| 197 |
-
if revoked:
|
| 198 |
-
connection.execute("UPDATE lab_route_library SET active=false, updated_at=now() "
|
| 199 |
-
"WHERE source=%s AND source_id = ANY(%s)", (SOURCE, list(revoked)))
|
| 200 |
-
if payload.collection_finalized:
|
| 201 |
-
connection.execute("UPDATE lab_route_library SET collection_finalized=true "
|
| 202 |
-
"WHERE source=%s AND active=true", (SOURCE,))
|
| 203 |
_invalidate_cache()
|
| 204 |
-
return
|
| 205 |
|
| 206 |
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Versioned HTTP ingestion. Runtime requests never execute schema DDL."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
|
|
|
|
|
|
|
|
|
| 4 |
import hmac
|
|
|
|
| 5 |
import json
|
|
|
|
| 6 |
import os
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
from threading import Lock
|
| 9 |
+
from uuid import UUID
|
| 10 |
|
| 11 |
import psycopg
|
| 12 |
+
from psycopg.rows import dict_row
|
| 13 |
+
from fastapi import APIRouter, Header, HTTPException, Request
|
| 14 |
+
from fastapi.exceptions import RequestValidationError
|
| 15 |
+
from fastapi.routing import APIRoute
|
| 16 |
|
| 17 |
from app.core.config import settings
|
| 18 |
+
from app.schemas.route_library import (
|
| 19 |
+
ApprovalSnapshot, ApprovalSnapshotV2, RouteImport, RouteImportV2, SessionStart,
|
| 20 |
+
)
|
| 21 |
+
from app.services import route_library_store as store
|
| 22 |
+
from app.services.art_route_ingest import (
|
| 23 |
+
SOURCE, LABELS, HASH as ID_PATTERN, MAX_GPX_BYTES, IngestError,
|
| 24 |
+
outline_from_points, parse_gpx, prepare_route,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class PrivateImportRoute(APIRoute):
|
| 29 |
+
def get_route_handler(self):
|
| 30 |
+
original = super().get_route_handler()
|
| 31 |
+
|
| 32 |
+
async def handler(request: Request):
|
| 33 |
+
if len(await request.body()) > 2 * 1024 * 1024:
|
| 34 |
+
raise HTTPException(413, detail="import_body_too_large")
|
| 35 |
+
try:
|
| 36 |
+
return await original(request)
|
| 37 |
+
except RequestValidationError:
|
| 38 |
+
# Default validation responses echo raw rejected payload values.
|
| 39 |
+
raise HTTPException(422, detail="invalid_route_library_request") from None
|
| 40 |
+
return handler
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
router = APIRouter(prefix="/ai/internal/route-library", tags=["route-library"], route_class=PrivateImportRoute)
|
| 44 |
_cache_lock = Lock()
|
| 45 |
+
_cache_revision = None
|
| 46 |
+
_cache = {}
|
| 47 |
+
DEPLOYMENT_REPORT_PATH = Path('/tmp/route-library-deployment-check.json')
|
| 48 |
+
DEPLOYMENT_SOURCE_PATH = Path(__file__).resolve().parents[4] / 'deployment-source.json'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
+
def _database_url():
|
|
|
|
| 52 |
url = os.environ.get("ROUTE_LIBRARY_DATABASE_URL", "")
|
| 53 |
if not url:
|
| 54 |
+
raise HTTPException(503, detail="route_library_not_configured")
|
| 55 |
return url.replace("postgresql+psycopg://", "postgresql://", 1)
|
| 56 |
|
| 57 |
|
| 58 |
+
def _authorize(authorization):
|
| 59 |
expected = os.environ.get("ROUTE_LIBRARY_TOKEN", "")
|
| 60 |
supplied = authorization.removeprefix("Bearer ") if authorization else ""
|
| 61 |
if not expected:
|
| 62 |
+
raise HTTPException(503, detail="route_library_disabled")
|
| 63 |
+
if not hmac.compare_digest(expected.encode(), supplied.encode()):
|
| 64 |
+
raise HTTPException(401, detail="invalid_route_library_authorization")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _execute(operation, readonly=False):
|
| 68 |
+
try:
|
| 69 |
+
options = "-c statement_timeout=15000 -c lock_timeout=5000"
|
| 70 |
+
if readonly:
|
| 71 |
+
options += " -c default_transaction_read_only=on"
|
| 72 |
+
with psycopg.connect(_database_url(), autocommit=True, row_factory=dict_row,
|
| 73 |
+
connect_timeout=5, options=options) as conn:
|
| 74 |
+
return operation(conn)
|
| 75 |
+
except IngestError as exc:
|
| 76 |
+
raise HTTPException(exc.status, detail=exc.code) from None
|
| 77 |
+
except psycopg.errors.UniqueViolation:
|
| 78 |
+
raise HTTPException(409, detail="source_identity_conflict") from None
|
| 79 |
+
except psycopg.Error:
|
| 80 |
+
# DB errors can contain DSNs, GPX values and complete SQL parameters.
|
| 81 |
+
raise HTTPException(503, detail="route_library_storage_unavailable") from None
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _outline(gpx, source_id):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
try:
|
| 86 |
+
return outline_from_points(parse_gpx(gpx), source_id)
|
| 87 |
+
except IngestError as exc:
|
| 88 |
+
raise HTTPException(exc.status, detail=exc.code) from None
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _invalidate_cache():
|
| 92 |
+
global _cache_revision, _cache
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
with _cache_lock:
|
| 94 |
+
_cache_revision, _cache = None, {}
|
| 95 |
|
| 96 |
|
| 97 |
@router.put("/routes/{source_id}")
|
| 98 |
+
def put_route(source_id: str, payload: RouteImport | RouteImportV2,
|
| 99 |
+
authorization: str | None = Header(default=None)):
|
| 100 |
_authorize(authorization)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
try:
|
| 102 |
+
prepared = prepare_route(source_id, payload)
|
| 103 |
+
except IngestError as exc:
|
| 104 |
+
raise HTTPException(exc.status, detail=exc.code) from None
|
| 105 |
+
operation = store.store_route if payload.contract_version == 2 else store.store_legacy
|
| 106 |
+
result = _execute(lambda conn: operation(conn, source_id, payload, prepared))
|
| 107 |
+
if result.get("result") != "unchanged":
|
| 108 |
+
_invalidate_cache()
|
| 109 |
+
return result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
|
| 112 |
@router.post("/approval-snapshot")
|
| 113 |
+
def apply_snapshot(payload: ApprovalSnapshot | ApprovalSnapshotV2,
|
| 114 |
+
authorization: str | None = Header(default=None)):
|
| 115 |
_authorize(authorization)
|
| 116 |
+
try:
|
| 117 |
+
store._validate_snapshot(payload)
|
| 118 |
+
except IngestError as exc:
|
| 119 |
+
raise HTTPException(exc.status, detail=exc.code) from None
|
| 120 |
+
result = _execute(lambda conn: store.apply_snapshot(conn, payload))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
_invalidate_cache()
|
| 122 |
+
return result
|
| 123 |
|
| 124 |
|
| 125 |
+
@router.get("/sync-state")
|
| 126 |
+
def get_sync_state(authorization: str | None = Header(default=None)):
|
| 127 |
+
_authorize(authorization)
|
| 128 |
+
return _execute(store.sync_state, readonly=True)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
@router.get('/deployment-status')
|
| 132 |
+
def deployment_status(authorization: str | None = Header(default=None)):
|
| 133 |
+
_authorize(authorization)
|
| 134 |
+
try:
|
| 135 |
+
report = json.loads(DEPLOYMENT_REPORT_PATH.read_text())
|
| 136 |
+
provenance = json.loads(DEPLOYMENT_SOURCE_PATH.read_text())
|
| 137 |
+
if report.get('ready') is not True or report.get('same_database') is not True:
|
| 138 |
+
raise ValueError
|
| 139 |
+
except (OSError, ValueError, TypeError):
|
| 140 |
+
raise HTTPException(503, detail='deployment_preflight_unavailable') from None
|
| 141 |
+
def verify(conn):
|
| 142 |
+
versions = [row['version_num'] for row in conn.execute('SELECT version_num FROM alembic_version')]
|
| 143 |
+
present = {name: bool(conn.execute('SELECT to_regclass(%s) AS relation', (name,)).fetchone()['relation'])
|
| 144 |
+
for name in ('lab_route_library', 'art_templates', 'art_route_candidates')}
|
| 145 |
+
return {'contract_version': 2, 'github_sha': provenance.get('github_sha'),
|
| 146 |
+
'same_database': True, 'catalog_uses_database': report.get('catalog_uses_database') is True,
|
| 147 |
+
'migration_revisions': versions, 'tables': present,
|
| 148 |
+
'migration_existing_rows_preserved': report.get('existing_rows_preserved') is True,
|
| 149 |
+
'ready': versions == ['0014_link_lab_art_routes'] and all(present.values())}
|
| 150 |
+
return _execute(verify, readonly=True)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
@router.get('/routes/{source_id}/storage')
|
| 154 |
+
def route_storage(source_id: str, authorization: str | None = Header(default=None)):
|
| 155 |
+
_authorize(authorization)
|
| 156 |
+
if not ID_PATTERN.fullmatch(source_id):
|
| 157 |
+
raise HTTPException(422, detail='invalid_source_id')
|
| 158 |
+
return _execute(lambda conn: store.verify_storage(conn, source_id), readonly=True)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
@router.post("/sync-sessions")
|
| 162 |
+
def start_session(payload: SessionStart, authorization: str | None = Header(default=None)):
|
| 163 |
+
_authorize(authorization)
|
| 164 |
+
return _execute(lambda conn: store.begin_session(conn, payload))
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
@router.get("/sync-sessions/{session_id}")
|
| 168 |
+
def get_session(session_id: UUID, authorization: str | None = Header(default=None)):
|
| 169 |
+
_authorize(authorization)
|
| 170 |
+
return _execute(lambda conn: store.session_status(conn, session_id), readonly=True)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
@router.post("/sync-sessions/{session_id}/complete")
|
| 174 |
+
def finish_partial(session_id: UUID, generation: int, authorization: str | None = Header(default=None)):
|
| 175 |
+
_authorize(authorization)
|
| 176 |
+
return _execute(lambda conn: store.complete_partial(conn, session_id, generation))
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def catalog_revision():
|
| 180 |
+
if not os.environ.get("ROUTE_LIBRARY_DATABASE_URL"):
|
| 181 |
+
return None
|
| 182 |
+
try:
|
| 183 |
+
return _execute(store.sync_state, readonly=True)["catalog_revision"]
|
| 184 |
+
except HTTPException:
|
| 185 |
+
return None
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def active_outlines():
|
| 189 |
+
"""Check a shared revision before caching; never replay external data on DB failure."""
|
| 190 |
+
global _cache_revision, _cache
|
| 191 |
+
if not os.environ.get("ROUTE_LIBRARY_DATABASE_URL"):
|
| 192 |
+
_invalidate_cache()
|
| 193 |
+
return {}
|
| 194 |
+
try:
|
| 195 |
+
def load(conn):
|
| 196 |
+
global _cache_revision, _cache
|
| 197 |
+
with conn.transaction():
|
| 198 |
+
conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
| 199 |
+
revision = store.sync_state(conn)["catalog_revision"]
|
| 200 |
+
with _cache_lock:
|
| 201 |
+
if revision == _cache_revision:
|
| 202 |
+
return _cache
|
| 203 |
+
rows = conn.execute("SELECT label,outline FROM lab_route_library WHERE source=%s AND active", (SOURCE,)).fetchall()
|
| 204 |
+
loaded = {}
|
| 205 |
+
for row in rows:
|
| 206 |
+
loaded.setdefault(row["label"], []).append(row["outline"])
|
| 207 |
+
with _cache_lock:
|
| 208 |
+
_cache_revision, _cache = revision, loaded
|
| 209 |
+
return loaded
|
| 210 |
+
return _execute(load, readonly=True)
|
| 211 |
+
except HTTPException:
|
| 212 |
+
_invalidate_cache()
|
| 213 |
+
return {}
|
api/ai-service/app/schemas/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""HTTP contracts for the AI service."""
|
api/ai-service/app/schemas/route_library.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Versioned imports. Coordinates are deliberately not an input field in v2."""
|
| 2 |
+
from typing import Annotated, Literal
|
| 3 |
+
from uuid import UUID
|
| 4 |
+
|
| 5 |
+
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
| 6 |
+
|
| 7 |
+
Digest = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{64}$")]
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class RouteImport(BaseModel):
|
| 11 |
+
contract_version: Literal[1] = 1
|
| 12 |
+
label: str
|
| 13 |
+
region_code: str = Field(max_length=100)
|
| 14 |
+
region_name: str = Field(max_length=200)
|
| 15 |
+
sha256: str
|
| 16 |
+
accepted_at: str
|
| 17 |
+
review_status: str
|
| 18 |
+
ai_protocol: str = Field(max_length=200)
|
| 19 |
+
geometry_ok: bool
|
| 20 |
+
seoul_ok: bool
|
| 21 |
+
collection_finalized: bool = False
|
| 22 |
+
gpx_base64: str = Field(max_length=175000)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class TemplateImport(BaseModel):
|
| 26 |
+
model_config = ConfigDict(extra="forbid")
|
| 27 |
+
design_id: str = Field(min_length=1, max_length=96, pattern=r"^[a-z0-9][a-z0-9_-]*$")
|
| 28 |
+
source_version: str = Field(min_length=1, max_length=100)
|
| 29 |
+
design_digest: Digest
|
| 30 |
+
snapshot: dict
|
| 31 |
+
design_ai_status: Literal["ai_pass", "ai_reject", "ai_review", "unknown"]
|
| 32 |
+
design_ai_evidence: dict = Field(default_factory=dict)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class RouteImportV2(BaseModel):
|
| 36 |
+
model_config = ConfigDict(extra="forbid")
|
| 37 |
+
contract_version: Literal[2]
|
| 38 |
+
session_id: UUID
|
| 39 |
+
generation: int = Field(gt=0)
|
| 40 |
+
payload_sha256: Digest
|
| 41 |
+
label: str
|
| 42 |
+
region_code: str = Field(min_length=1, max_length=40)
|
| 43 |
+
region_name: str = Field(min_length=1, max_length=200)
|
| 44 |
+
sha256: Digest
|
| 45 |
+
accepted_at: str
|
| 46 |
+
review_status: Literal["ai_pass"]
|
| 47 |
+
ai_protocol: str = Field(min_length=1, max_length=200)
|
| 48 |
+
geometry_ok: Literal[True]
|
| 49 |
+
seoul_ok: Literal[True]
|
| 50 |
+
collection_finalized: bool = False
|
| 51 |
+
gpx_base64: str = Field(max_length=175000)
|
| 52 |
+
template: TemplateImport
|
| 53 |
+
route_ai_evidence: dict
|
| 54 |
+
road_verification: dict
|
| 55 |
+
graph_sha256: Digest
|
| 56 |
+
distance_m: int = Field(gt=0, lt=2147483648)
|
| 57 |
+
validator_version: str = Field(min_length=1, max_length=80)
|
| 58 |
+
previous_payload_sha256: Digest | None = None
|
| 59 |
+
reapprove: bool = False
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class ApprovalSnapshot(BaseModel):
|
| 63 |
+
contract_version: Literal[1] = 1
|
| 64 |
+
ids: list[str] = Field(max_length=10000)
|
| 65 |
+
count: int = Field(ge=0, le=10000)
|
| 66 |
+
ids_sha256: str
|
| 67 |
+
collection_finalized: bool = False
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class ApprovalSnapshotV2(ApprovalSnapshot):
|
| 71 |
+
model_config = ConfigDict(extra="forbid")
|
| 72 |
+
contract_version: Literal[2]
|
| 73 |
+
session_id: UUID
|
| 74 |
+
generation: int = Field(gt=0)
|
| 75 |
+
manifest_sha256: Digest
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
class SessionStart(BaseModel):
|
| 79 |
+
model_config = ConfigDict(extra="forbid")
|
| 80 |
+
contract_version: Literal[2] = 2
|
| 81 |
+
client_request_id: UUID
|
| 82 |
+
expected_generation: int = Field(ge=0)
|
| 83 |
+
mode: Literal["full", "partial"]
|
| 84 |
+
manifest: dict[Digest, Digest] = Field(max_length=10000)
|
| 85 |
+
manifest_sha256: Digest
|
api/ai-service/app/services/art_route_catalog.py
CHANGED
|
@@ -1,19 +1,19 @@
|
|
| 1 |
"""Read validated GPS exemplars, never serve their original GPS as a new route."""
|
| 2 |
-
from functools import lru_cache
|
| 3 |
import json
|
| 4 |
import logging
|
| 5 |
import math
|
| 6 |
import os
|
| 7 |
from pathlib import Path
|
| 8 |
-
import time
|
| 9 |
|
| 10 |
from app.services.gpx_shape_similarity import _sample_outline
|
| 11 |
|
| 12 |
LOG = logging.getLogger(__name__)
|
| 13 |
CATALOG_SQL = """
|
| 14 |
-
SELECT t.template_key, t.name, r.coordinates
|
| 15 |
FROM art_route_candidates r JOIN art_templates t ON t.id = r.template_id
|
| 16 |
-
|
|
|
|
|
|
|
| 17 |
ORDER BY r.validation_score DESC NULLS LAST, r.created_at DESC, r.id
|
| 18 |
LIMIT 200
|
| 19 |
"""
|
|
@@ -45,28 +45,33 @@ def coordinate_outline(coordinates, order="latlon"):
|
|
| 45 |
return [*outline, outline[0]] if outline else None
|
| 46 |
|
| 47 |
|
| 48 |
-
|
| 49 |
-
def _load(dsn, filename, order, _refresh_bucket):
|
| 50 |
try:
|
| 51 |
if filename:
|
| 52 |
records = json.loads(Path(filename).read_text(encoding="utf-8"))
|
| 53 |
if not isinstance(records, list):
|
| 54 |
raise ValueError("Expected a list of route records")
|
| 55 |
-
rows = [(r.get("template_key"), r.get("name"), r.get("coordinates"))
|
| 56 |
-
for r in records[:200] if isinstance(r, dict) and r.get("validation_status") == "passed"]
|
| 57 |
elif dsn:
|
| 58 |
import psycopg
|
| 59 |
-
with psycopg.connect(dsn, connect_timeout=3,
|
| 60 |
options="-c default_transaction_read_only=on -c statement_timeout=3000") as conn:
|
| 61 |
rows = conn.execute(CATALOG_SQL).fetchall()
|
| 62 |
else:
|
| 63 |
return {}
|
| 64 |
catalog = {}
|
| 65 |
-
|
|
|
|
| 66 |
if not isinstance(key, str) or not key.strip():
|
| 67 |
continue
|
| 68 |
-
outline
|
|
|
|
| 69 |
if outline:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
entry = catalog.setdefault(key, (str(name or key), []))
|
| 71 |
entry[1].append(outline)
|
| 72 |
return catalog
|
|
@@ -81,5 +86,7 @@ def load_art_route_catalog():
|
|
| 81 |
if order not in {"latlon", "lonlat"}:
|
| 82 |
LOG.warning("Invalid ART_ROUTE_COORDINATE_ORDER; ignoring external catalog")
|
| 83 |
return {}
|
| 84 |
-
|
| 85 |
-
|
|
|
|
|
|
|
|
|
| 1 |
"""Read validated GPS exemplars, never serve their original GPS as a new route."""
|
|
|
|
| 2 |
import json
|
| 3 |
import logging
|
| 4 |
import math
|
| 5 |
import os
|
| 6 |
from pathlib import Path
|
|
|
|
| 7 |
|
| 8 |
from app.services.gpx_shape_similarity import _sample_outline
|
| 9 |
|
| 10 |
LOG = logging.getLogger(__name__)
|
| 11 |
CATALOG_SQL = """
|
| 12 |
+
SELECT t.template_key, t.name, r.coordinates, r.coordinate_order, l.outline
|
| 13 |
FROM art_route_candidates r JOIN art_templates t ON t.id = r.template_id
|
| 14 |
+
LEFT JOIN lab_route_library l ON l.source=r.source AND l.source_id=r.source_id
|
| 15 |
+
WHERE r.validation_status = 'passed' AND r.active
|
| 16 |
+
AND (r.source IS NULL OR (l.active AND l.art_projection_version=2 AND l.payload_sha256=r.payload_sha256))
|
| 17 |
ORDER BY r.validation_score DESC NULLS LAST, r.created_at DESC, r.id
|
| 18 |
LIMIT 200
|
| 19 |
"""
|
|
|
|
| 45 |
return [*outline, outline[0]] if outline else None
|
| 46 |
|
| 47 |
|
| 48 |
+
def _load(dsn, filename, order):
|
|
|
|
| 49 |
try:
|
| 50 |
if filename:
|
| 51 |
records = json.loads(Path(filename).read_text(encoding="utf-8"))
|
| 52 |
if not isinstance(records, list):
|
| 53 |
raise ValueError("Expected a list of route records")
|
| 54 |
+
rows = [(r.get("template_key"), r.get("name"), r.get("coordinates"), r.get("coordinate_order"), None)
|
| 55 |
+
for r in records[:200] if isinstance(r, dict) and r.get("validation_status") == "passed" and r.get("active", True)]
|
| 56 |
elif dsn:
|
| 57 |
import psycopg
|
| 58 |
+
with psycopg.connect(dsn.replace("postgresql+psycopg://", "postgresql://", 1), connect_timeout=3,
|
| 59 |
options="-c default_transaction_read_only=on -c statement_timeout=3000") as conn:
|
| 60 |
rows = conn.execute(CATALOG_SQL).fetchall()
|
| 61 |
else:
|
| 62 |
return {}
|
| 63 |
catalog = {}
|
| 64 |
+
seen = {}
|
| 65 |
+
for key, name, coordinates, coordinate_order, imported_outline in rows:
|
| 66 |
if not isinstance(key, str) or not key.strip():
|
| 67 |
continue
|
| 68 |
+
# A library outline is already XY; never pass it through GPS parsing.
|
| 69 |
+
outline = imported_outline if imported_outline is not None else coordinate_outline(coordinates, coordinate_order or order)
|
| 70 |
if outline:
|
| 71 |
+
signature = tuple(map(tuple, outline))
|
| 72 |
+
if signature in seen.setdefault(key, set()):
|
| 73 |
+
continue
|
| 74 |
+
seen[key].add(signature)
|
| 75 |
entry = catalog.setdefault(key, (str(name or key), []))
|
| 76 |
entry[1].append(outline)
|
| 77 |
return catalog
|
|
|
|
| 86 |
if order not in {"latlon", "lonlat"}:
|
| 87 |
LOG.warning("Invalid ART_ROUTE_COORDINATE_ORDER; ignoring external catalog")
|
| 88 |
return {}
|
| 89 |
+
# Database results are fresh on every call; a previous process-local cache
|
| 90 |
+
# must not survive an approval revocation made by another API process.
|
| 91 |
+
return _load(os.getenv("ART_ROUTE_CATALOG_DATABASE_URL") or os.getenv("ROUTE_LIBRARY_DATABASE_URL"),
|
| 92 |
+
os.getenv("ART_ROUTE_CATALOG_JSON"), order)
|
api/ai-service/app/services/art_route_ingest.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure CPU validation/derivation. Never read paths provided in an HTTP payload."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import base64
|
| 5 |
+
import binascii
|
| 6 |
+
from datetime import datetime
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import math
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import re
|
| 12 |
+
import sys
|
| 13 |
+
import uuid
|
| 14 |
+
import xml.etree.ElementTree as ET
|
| 15 |
+
|
| 16 |
+
from app.core.config import settings
|
| 17 |
+
|
| 18 |
+
SOURCE = "lab-national-ai-seven-100-20261106"
|
| 19 |
+
LABELS = {"dog": "dog", "heart": "heart", "tree": "christmas_tree_trunk",
|
| 20 |
+
"tshirt": "t_shirt", "candle": "candle", "pants": "long_pants", "star": "symbol_star"}
|
| 21 |
+
NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://github.com/hwangtutu/RunTheShape/lab-art-import/v1")
|
| 22 |
+
HASH = re.compile(r"^[0-9a-f]{64}$")
|
| 23 |
+
MAX_GPX_BYTES = 128 * 1024
|
| 24 |
+
SVG_VERSION = "canonical-design-points-y-up-to-svg-v1"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class IngestError(Exception):
|
| 28 |
+
def __init__(self, code, status=422):
|
| 29 |
+
self.code, self.status = code, status
|
| 30 |
+
super().__init__(code)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def require(condition, code, status=422):
|
| 34 |
+
if not condition:
|
| 35 |
+
raise IngestError(code, status)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def canonical(value):
|
| 39 |
+
try:
|
| 40 |
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
|
| 41 |
+
except (ValueError, TypeError, RecursionError):
|
| 42 |
+
raise IngestError("invalid_json_value") from None
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def sha(value):
|
| 46 |
+
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def stable_uuid(kind, *parts):
|
| 50 |
+
return uuid.uuid5(NAMESPACE, canonical([kind, SOURCE, *parts]))
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def content_sha(payload):
|
| 54 |
+
# Preserve nonfinite floats until canonical() rejects them; JSON-mode model
|
| 55 |
+
# serialization may otherwise turn a rejected NaN into an ordinary null.
|
| 56 |
+
data = payload.model_dump(mode="python") if hasattr(payload, "model_dump") else dict(payload)
|
| 57 |
+
for key in ("session_id", "generation", "payload_sha256", "gpx_base64",
|
| 58 |
+
"previous_payload_sha256", "reapprove"):
|
| 59 |
+
data.pop(key, None)
|
| 60 |
+
return sha(data)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def accepted_time(value):
|
| 64 |
+
try:
|
| 65 |
+
result = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 66 |
+
require(result.tzinfo is not None, "invalid_accepted_at")
|
| 67 |
+
return result
|
| 68 |
+
except (TypeError, ValueError, OverflowError):
|
| 69 |
+
raise IngestError("invalid_accepted_at") from None
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def parse_gpx(raw):
|
| 73 |
+
try:
|
| 74 |
+
require(0 < len(raw) <= MAX_GPX_BYTES, "gpx_size", 413)
|
| 75 |
+
require(b"<!DOCTYPE" not in raw.upper() and b"<!ENTITY" not in raw.upper(), "invalid_gpx_xml")
|
| 76 |
+
root = ET.fromstring(raw)
|
| 77 |
+
require(root.tag.rsplit("}", 1)[-1] == "gpx", "invalid_gpx_document")
|
| 78 |
+
tracks = [n for n in root.iter() if n.tag.rsplit("}", 1)[-1] == "trk"]
|
| 79 |
+
segments = [n for n in root.iter() if n.tag.rsplit("}", 1)[-1] == "trkseg"]
|
| 80 |
+
require(len(tracks) == len(segments) == 1 and segments[0] in list(tracks[0]), "invalid_gpx_segments")
|
| 81 |
+
points = [[float(n.attrib["lat"]), float(n.attrib["lon"])]
|
| 82 |
+
for n in segments[0] if n.tag.rsplit("}", 1)[-1] == "trkpt"]
|
| 83 |
+
require(12 <= len(points) <= 10000, "invalid_gpx_point_count")
|
| 84 |
+
require(all(math.isfinite(a) and math.isfinite(b) and abs(a) <= 90 and abs(b) <= 180
|
| 85 |
+
for a, b in points), "invalid_gpx_coordinates")
|
| 86 |
+
require(len(set(map(tuple, points))) >= 4 and math.dist(points[0], points[-1]) <= 1e-5, "invalid_gpx_closure")
|
| 87 |
+
return points
|
| 88 |
+
except (ET.ParseError, KeyError, ValueError, TypeError, OverflowError):
|
| 89 |
+
raise IngestError("invalid_gpx") from None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def outline_from_points(points, name):
|
| 93 |
+
scripts = str(Path(settings.ai_workspace_dir) / "scripts")
|
| 94 |
+
if scripts not in sys.path:
|
| 95 |
+
sys.path.insert(0, scripts)
|
| 96 |
+
from build_gpx_shape_templates import normalized_outline_from_track
|
| 97 |
+
try:
|
| 98 |
+
return normalized_outline_from_track(points, name)
|
| 99 |
+
except (ValueError, RecursionError, OverflowError):
|
| 100 |
+
raise IngestError("invalid_gpx_outline") from None
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def category(text):
|
| 104 |
+
value = text.lower().strip()
|
| 105 |
+
if re.search(r"\b(unknown|unrecognizable|nothing|blank|abstract|polygon|scribble|geometric)\b", value):
|
| 106 |
+
return "unknown"
|
| 107 |
+
patterns = [("shorts", r"\bshorts\b|short[ -](trousers|pants)"),
|
| 108 |
+
("sweater", r"long[ -]sleeve|\b(sweater|jacket|coat)\b"),
|
| 109 |
+
("heart", r"\bheart\b|love[ -](symbol|icon)|\bvalentine\b"),
|
| 110 |
+
("tree", r"\b(tree|fir|pine|spruce)\b"), ("tshirt", r"\bt[ -]?shirt\b|\btee\b|\bshirt\b"),
|
| 111 |
+
("pants", r"\b(pants|trousers?|jeans|slacks)\b"), ("candle", r"\bcandle\b"),
|
| 112 |
+
("dog", r"\b(dog|puppy|hound|canine|terrier|dachshund|poodle|corgi|beagle|labrador|retriever|bulldog|dalmatian|husky|shiba|schnauzer|chihuahua)\b"),
|
| 113 |
+
("cat", r"\b(cat|kitten|feline)\b"), ("circle", r"\b(circle|oval)\b"),
|
| 114 |
+
("square", r"\b(square|rectangle)\b")]
|
| 115 |
+
for label, pattern in patterns:
|
| 116 |
+
if re.search(pattern, value):
|
| 117 |
+
return label
|
| 118 |
+
return "star" if re.search(r"\bstars?\b", value) else "other"
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def valid_route_review(review, label, protocol):
|
| 122 |
+
try:
|
| 123 |
+
views, responses = review["siglip_views"], review["qwen"]
|
| 124 |
+
return (review.get("status") == "ai_pass" and review.get("expected_label") == label
|
| 125 |
+
and review.get("protocol") == protocol and review.get("blind") is True
|
| 126 |
+
and review.get("qwen_performed") is True and bool(review.get("siglip_support_angles"))
|
| 127 |
+
and isinstance(views, list) and any(isinstance(v, dict) and v.get("top_label") == label for v in views)
|
| 128 |
+
and isinstance(responses, list) and bool(responses)
|
| 129 |
+
and all(isinstance(q, dict) and q.get("valid") is True and isinstance(q.get("object"), str)
|
| 130 |
+
and type(q.get("clarity")) is int and 0 <= q["clarity"] <= 3 for q in responses)
|
| 131 |
+
and any(category(q["object"]) == label and q["clarity"] >= 2 for q in responses))
|
| 132 |
+
except (KeyError, TypeError, AttributeError):
|
| 133 |
+
return False
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def render_template(template):
|
| 137 |
+
snapshot = template.snapshot
|
| 138 |
+
require(len(canonical(snapshot).encode()) <= 512 * 1024, "design_size", 413)
|
| 139 |
+
require(sha(snapshot) == template.design_digest, "design_digest_mismatch")
|
| 140 |
+
require(snapshot.get("design_id") == template.design_id and str(snapshot.get("version")) == template.source_version,
|
| 141 |
+
"design_identity_mismatch")
|
| 142 |
+
name, points = snapshot.get("name"), snapshot.get("points")
|
| 143 |
+
require(isinstance(name, str) and 0 < len(name) <= 200, "design_name")
|
| 144 |
+
require(isinstance(snapshot.get("outline"), dict)
|
| 145 |
+
and snapshot["outline"].get("coordinate_system") == "cartesian_y_up", "design_coordinate_system")
|
| 146 |
+
require(isinstance(points, list) and 3 <= len(points) <= 10000, "design_point_count")
|
| 147 |
+
require(all(isinstance(p, list) and len(p) == 2 and all(type(v) in (int, float) and math.isfinite(v) for v in p)
|
| 148 |
+
for p in points), "design_points")
|
| 149 |
+
require(len(set(map(tuple, points))) >= 3, "design_distinct_points")
|
| 150 |
+
if "status" in template.design_ai_evidence:
|
| 151 |
+
require(template.design_ai_evidence["status"] == template.design_ai_status, "design_review_status_mismatch")
|
| 152 |
+
lo = [min(p[j] for p in points) for j in (0, 1)]
|
| 153 |
+
hi = [max(p[j] for p in points) for j in (0, 1)]
|
| 154 |
+
extent = max(hi[j] - lo[j] for j in (0, 1))
|
| 155 |
+
require(math.isfinite(extent) and extent > 0, "design_extent")
|
| 156 |
+
center = [lo[j] + (hi[j] - lo[j]) / 2 for j in (0, 1)]
|
| 157 |
+
body = [((x - center[0]) / extent * 900 + 500, (center[1] - y) / extent * 900 + 500) for x, y in points]
|
| 158 |
+
ns = "http://www.w3.org/2000/svg"
|
| 159 |
+
ET.register_namespace("", ns)
|
| 160 |
+
svg = ET.Element(f"{{{ns}}}svg", {"viewBox": "0 0 1000 1000", "version": "1.1"})
|
| 161 |
+
ET.SubElement(svg, f"{{{ns}}}title").text = name
|
| 162 |
+
ET.SubElement(svg, f"{{{ns}}}metadata").text = canonical({"source": SOURCE, "design_id": template.design_id,
|
| 163 |
+
"source_version": template.source_version, "design_digest": template.design_digest,
|
| 164 |
+
"derivation": SVG_VERSION})
|
| 165 |
+
ET.SubElement(svg, f"{{{ns}}}path", {"d": "M " + " L ".join(f"{x:.9f},{y:.9f}" for x, y in body) + " Z",
|
| 166 |
+
"fill": "none", "stroke": "currentColor", "stroke-width": "10", "stroke-linejoin": "round"})
|
| 167 |
+
return name, ET.tostring(svg, encoding="unicode")
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def prepare_route(source_id, payload):
|
| 171 |
+
require(HASH.fullmatch(source_id) and payload.label in LABELS, "unknown_route_id_or_label")
|
| 172 |
+
require(payload.review_status == "ai_pass" and payload.geometry_ok and payload.seoul_ok, "route_not_approved")
|
| 173 |
+
accepted_at = accepted_time(payload.accepted_at)
|
| 174 |
+
try:
|
| 175 |
+
gpx = base64.b64decode(payload.gpx_base64, validate=True)
|
| 176 |
+
except (ValueError, binascii.Error):
|
| 177 |
+
raise IngestError("invalid_gpx_base64") from None
|
| 178 |
+
require(hashlib.sha256(gpx).hexdigest() == payload.sha256.lower(), "GPX SHA-256 mismatch")
|
| 179 |
+
points = parse_gpx(gpx)
|
| 180 |
+
result = {"gpx": gpx, "coordinates": points, "outline": outline_from_points(points, source_id),
|
| 181 |
+
"accepted_at": accepted_at, "label": LABELS[payload.label]}
|
| 182 |
+
if payload.contract_version == 2:
|
| 183 |
+
require(content_sha(payload) == payload.payload_sha256, "payload_digest_mismatch")
|
| 184 |
+
require(len(canonical(payload.model_dump(mode="json")).encode()) <= 2 * 1024 * 1024, "payload_size", 413)
|
| 185 |
+
require(valid_route_review(payload.route_ai_evidence, payload.label, payload.ai_protocol), "route_ai_evidence")
|
| 186 |
+
if payload.template.design_ai_status == "ai_pass":
|
| 187 |
+
evidence = payload.template.design_ai_evidence
|
| 188 |
+
require(bool(evidence.get("protocol")) and valid_route_review(evidence, payload.label, evidence["protocol"]),
|
| 189 |
+
"design_ai_evidence")
|
| 190 |
+
proof = payload.road_verification
|
| 191 |
+
require(all(proof.get(k) is True for k in ("road_only", "graph_connectivity_verified", "access_checked", "directions_checked"))
|
| 192 |
+
and type(proof.get("air_length_m")) in (int, float) and proof["air_length_m"] == 0, "road_verification")
|
| 193 |
+
result["name"], result["svg"] = render_template(payload.template)
|
| 194 |
+
result["template_id"] = stable_uuid("template", payload.label, payload.template.design_id, payload.template.design_digest)
|
| 195 |
+
result["candidate_id"] = stable_uuid("route", source_id)
|
| 196 |
+
return result
|
api/ai-service/app/services/route_library_store.py
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One transaction per import or snapshot; every writer locks the source first."""
|
| 2 |
+
from uuid import uuid4
|
| 3 |
+
|
| 4 |
+
from psycopg.types.json import Jsonb
|
| 5 |
+
|
| 6 |
+
from app.services.art_route_ingest import SOURCE, SVG_VERSION, require, sha, parse_gpx
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _lock_source(conn):
|
| 10 |
+
conn.execute("INSERT INTO lab_route_sync_state(source) VALUES (%s) ON CONFLICT DO NOTHING", (SOURCE,))
|
| 11 |
+
return conn.execute("SELECT * FROM lab_route_sync_state WHERE source=%s FOR UPDATE", (SOURCE,)).fetchone()
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _bump(conn):
|
| 15 |
+
conn.execute("UPDATE lab_route_sync_state SET catalog_revision=catalog_revision+1, updated_at=now() WHERE source=%s", (SOURCE,))
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def sync_state(conn):
|
| 19 |
+
row = conn.execute("SELECT generation,catalog_revision,v2_required FROM lab_route_sync_state WHERE source=%s", (SOURCE,)).fetchone()
|
| 20 |
+
return {"contract_version": 2, **(dict(row) if row else {"generation": 0, "catalog_revision": 0, "v2_required": False})}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _session_receipt(row):
|
| 24 |
+
return {"contract_version": 2, "session_id": str(row["id"]), "generation": row["generation"],
|
| 25 |
+
"mode": row["mode"], "session_status": row["status"], "manifest_sha256": row["manifest_sha256"]}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def begin_session(conn, payload):
|
| 29 |
+
require(sha(payload.manifest) == payload.manifest_sha256, "manifest_digest_mismatch")
|
| 30 |
+
with conn.transaction():
|
| 31 |
+
state = _lock_source(conn)
|
| 32 |
+
existing = conn.execute("SELECT * FROM lab_route_sync_sessions WHERE source=%s AND client_request_id=%s",
|
| 33 |
+
(SOURCE, payload.client_request_id)).fetchone()
|
| 34 |
+
if existing:
|
| 35 |
+
require(existing["manifest_sha256"] == payload.manifest_sha256 and existing["mode"] == payload.mode
|
| 36 |
+
and existing["generation"] == payload.expected_generation + 1, "session_request_conflict", 409)
|
| 37 |
+
require(state["current_session_id"] == existing["id"], "stale_sync_session", 409)
|
| 38 |
+
return _session_receipt(existing)
|
| 39 |
+
require(state["generation"] == payload.expected_generation, "stale_sync_generation", 409)
|
| 40 |
+
if state["current_session_id"]:
|
| 41 |
+
conn.execute("UPDATE lab_route_sync_sessions SET status='superseded' WHERE id=%s AND status='open'",
|
| 42 |
+
(state["current_session_id"],))
|
| 43 |
+
session_id, generation = uuid4(), state["generation"] + 1
|
| 44 |
+
row = conn.execute("""INSERT INTO lab_route_sync_sessions
|
| 45 |
+
(id,source,client_request_id,generation,mode,status,manifest,manifest_sha256)
|
| 46 |
+
VALUES (%s,%s,%s,%s,%s,'open',%s,%s) RETURNING *""",
|
| 47 |
+
(session_id, SOURCE, payload.client_request_id, generation, payload.mode,
|
| 48 |
+
Jsonb(payload.manifest), payload.manifest_sha256)).fetchone()
|
| 49 |
+
conn.execute("UPDATE lab_route_sync_state SET generation=%s,current_session_id=%s,v2_required=true,updated_at=now() WHERE source=%s",
|
| 50 |
+
(generation, session_id, SOURCE))
|
| 51 |
+
result = _session_receipt(row)
|
| 52 |
+
return result
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _current_session(conn, state, session_id, generation):
|
| 56 |
+
require(state["current_session_id"] == session_id and state["generation"] == generation, "stale_sync_session", 409)
|
| 57 |
+
row = conn.execute("SELECT * FROM lab_route_sync_sessions WHERE id=%s AND source=%s", (session_id, SOURCE)).fetchone()
|
| 58 |
+
require(row is not None and row["status"] in ("open", "completed"), "stale_sync_session", 409)
|
| 59 |
+
return row
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def session_status(conn, session_id):
|
| 63 |
+
state = conn.execute("SELECT * FROM lab_route_sync_state WHERE source=%s", (SOURCE,)).fetchone()
|
| 64 |
+
require(state is not None and state["current_session_id"] == session_id, "stale_sync_session", 409)
|
| 65 |
+
row = _current_session(conn, state, session_id, state["generation"])
|
| 66 |
+
return _session_receipt(row)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def verify_storage(conn, source_id):
|
| 70 |
+
"""Read actual persisted links and compare GPS inside the server only."""
|
| 71 |
+
with conn.transaction():
|
| 72 |
+
conn.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY')
|
| 73 |
+
row = conn.execute('''SELECT l.gpx,l.gpx_sha256,l.payload_sha256 AS library_hash,l.active AS library_active,
|
| 74 |
+
l.art_projection_version,r.id AS candidate_id,r.template_id,r.active AS candidate_active,
|
| 75 |
+
r.payload_sha256 AS candidate_hash,r.gpx_sha256 AS candidate_gpx_hash,r.coordinates,
|
| 76 |
+
r.coordinate_order,r.coordinate_crs,r.validation_status,r.route_ai_status,
|
| 77 |
+
t.id AS stored_template_id,t.design_ai_status
|
| 78 |
+
FROM lab_route_library l
|
| 79 |
+
LEFT JOIN art_route_candidates r ON r.source=l.source AND r.source_id=l.source_id
|
| 80 |
+
LEFT JOIN art_templates t ON t.id=r.template_id
|
| 81 |
+
WHERE l.source=%s AND l.source_id=%s''', (SOURCE, source_id)).fetchone()
|
| 82 |
+
require(row is not None, 'route_storage_not_found', 404)
|
| 83 |
+
gps_matches = bool(row['candidate_id'] and row['coordinates'] == parse_gpx(bytes(row['gpx'])))
|
| 84 |
+
tables = {'lab_route_library': True, 'art_templates': row['stored_template_id'] is not None,
|
| 85 |
+
'art_route_candidates': row['candidate_id'] is not None}
|
| 86 |
+
linked = bool(all(tables.values()) and row['art_projection_version'] == 2
|
| 87 |
+
and row['library_hash'] == row['candidate_hash']
|
| 88 |
+
and row['gpx_sha256'] == row['candidate_gpx_hash'] and row['library_active']
|
| 89 |
+
and row['candidate_active'] and row['validation_status'] == 'passed'
|
| 90 |
+
and row['route_ai_status'] == 'ai_pass' and gps_matches
|
| 91 |
+
and row['coordinate_order'] == 'latlon' and row['coordinate_crs'] == 'EPSG:4326')
|
| 92 |
+
return {'contract_version': 2, 'source_id': source_id, 'verified': linked, 'tables': tables,
|
| 93 |
+
'art_template_id': str(row['stored_template_id']) if row['stored_template_id'] else None,
|
| 94 |
+
'art_route_candidate_id': str(row['candidate_id']) if row['candidate_id'] else None,
|
| 95 |
+
'sha256': row['gpx_sha256'], 'payload_sha256': row['library_hash'],
|
| 96 |
+
'design_ai_status': row['design_ai_status'], 'route_validation_status': row['validation_status'],
|
| 97 |
+
'coordinates_match_gpx': gps_matches, 'coordinate_order': row['coordinate_order'],
|
| 98 |
+
'coordinate_crs': row['coordinate_crs'], 'library_active': row['library_active'],
|
| 99 |
+
'candidate_active': row['candidate_active']}
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _put_library(conn, source_id, payload, prepared, content_hash, generation, projection):
|
| 103 |
+
conn.execute("""INSERT INTO lab_route_library
|
| 104 |
+
(source,source_id,label,region_code,region_name,gpx_sha256,accepted_at,ai_protocol,
|
| 105 |
+
collection_finalized,gpx,outline,active,payload_sha256,sync_generation,art_projection_version)
|
| 106 |
+
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,true,%s,%s,%s)
|
| 107 |
+
ON CONFLICT(source,source_id) DO UPDATE SET label=excluded.label,
|
| 108 |
+
region_code=excluded.region_code,region_name=excluded.region_name,gpx_sha256=excluded.gpx_sha256,
|
| 109 |
+
accepted_at=excluded.accepted_at,ai_protocol=excluded.ai_protocol,
|
| 110 |
+
collection_finalized=excluded.collection_finalized,gpx=excluded.gpx,outline=excluded.outline,
|
| 111 |
+
active=true,payload_sha256=excluded.payload_sha256,sync_generation=excluded.sync_generation,
|
| 112 |
+
art_projection_version=excluded.art_projection_version,revoked_at=NULL,revoked_generation=NULL,updated_at=now()""",
|
| 113 |
+
(SOURCE, source_id, prepared["label"], payload.region_code, payload.region_name, payload.sha256.lower(),
|
| 114 |
+
prepared["accepted_at"], payload.ai_protocol, payload.collection_finalized, prepared["gpx"],
|
| 115 |
+
Jsonb(prepared["outline"]), content_hash, generation, projection))
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _template(conn, payload, prepared):
|
| 119 |
+
template = payload.template
|
| 120 |
+
row = conn.execute("""SELECT * FROM art_templates WHERE source=%s AND source_label=%s
|
| 121 |
+
AND source_design_id=%s AND design_digest=%s""",
|
| 122 |
+
(SOURCE, payload.label, template.design_id, template.design_digest)).fetchone()
|
| 123 |
+
if row:
|
| 124 |
+
require(row["source_design_version"] == template.source_version and row["source_snapshot"] == template.snapshot
|
| 125 |
+
and row["name"] == prepared["name"] and row["svg"] == prepared["svg"]
|
| 126 |
+
and row["svg_render_version"] == SVG_VERSION, "template_content_conflict", 409)
|
| 127 |
+
if row["design_ai_status"] != template.design_ai_status or row["design_ai_evidence"] != template.design_ai_evidence:
|
| 128 |
+
require((row["design_review_generation"] or 0) < payload.generation, "template_review_conflict", 409)
|
| 129 |
+
row = conn.execute("""UPDATE art_templates SET design_ai_status=%s,design_ai_evidence=%s,
|
| 130 |
+
design_review_generation=%s WHERE id=%s RETURNING *""",
|
| 131 |
+
(template.design_ai_status, Jsonb(template.design_ai_evidence), payload.generation, row["id"])).fetchone()
|
| 132 |
+
return row
|
| 133 |
+
# Key/version is global, including rows from other sources. Allocate under a key lock.
|
| 134 |
+
conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", ("art-template:" + prepared["label"],))
|
| 135 |
+
version = conn.execute("SELECT COALESCE(max(version)::bigint,0)+1 AS version FROM art_templates WHERE template_key=%s",
|
| 136 |
+
(prepared["label"],)).fetchone()["version"]
|
| 137 |
+
require(version <= 2147483647, "template_version_exhausted", 409)
|
| 138 |
+
return conn.execute("""INSERT INTO art_templates
|
| 139 |
+
(id,template_key,name,version,svg,source,source_label,source_design_id,source_design_version,
|
| 140 |
+
design_digest,source_snapshot,design_ai_status,design_ai_evidence,svg_render_version,design_review_generation)
|
| 141 |
+
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *""",
|
| 142 |
+
(prepared["template_id"], prepared["label"], prepared["name"], version, prepared["svg"], SOURCE,
|
| 143 |
+
payload.label, template.design_id, template.source_version, template.design_digest,
|
| 144 |
+
Jsonb(template.snapshot), template.design_ai_status, Jsonb(template.design_ai_evidence), SVG_VERSION, payload.generation)).fetchone()
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _receipt(source_id, payload, template, candidate, result):
|
| 148 |
+
return {"status": "stored", "contract_version": 2, "result": result,
|
| 149 |
+
"source_id": source_id, "sha256": payload.sha256, "payload_sha256": payload.payload_sha256,
|
| 150 |
+
"art_template_id": str(template["id"]), "art_route_candidate_id": str(candidate["id"]),
|
| 151 |
+
"design_ai_status": template["design_ai_status"], "route_validation_status": candidate["validation_status"]}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def store_route(conn, source_id, payload, prepared):
|
| 155 |
+
with conn.transaction():
|
| 156 |
+
state = _lock_source(conn)
|
| 157 |
+
session = _current_session(conn, state, payload.session_id, payload.generation)
|
| 158 |
+
require(session["manifest"].get(source_id) == payload.payload_sha256, "route_not_in_manifest", 409)
|
| 159 |
+
old = conn.execute("SELECT * FROM lab_route_library WHERE source=%s AND source_id=%s", (SOURCE, source_id)).fetchone()
|
| 160 |
+
candidate = conn.execute("SELECT * FROM art_route_candidates WHERE source=%s AND source_id=%s", (SOURCE, source_id)).fetchone()
|
| 161 |
+
unchanged = bool(old and old["active"] and old["payload_sha256"] == payload.payload_sha256)
|
| 162 |
+
if not unchanged:
|
| 163 |
+
require(session["status"] == "open", "sync_session_completed", 409)
|
| 164 |
+
if old and not old["active"]:
|
| 165 |
+
require(payload.reapprove and prepared["accepted_at"] > old["accepted_at"], "explicit_reapproval_required", 409)
|
| 166 |
+
if old and old["art_projection_version"] >= 2 and old["payload_sha256"] != payload.payload_sha256:
|
| 167 |
+
require(payload.previous_payload_sha256 == old["payload_sha256"], "route_content_conflict", 409)
|
| 168 |
+
if old and old["art_projection_version"] < 2:
|
| 169 |
+
require(old["gpx_sha256"] == payload.sha256, "legacy_route_content_conflict", 409)
|
| 170 |
+
template = _template(conn, payload, prepared)
|
| 171 |
+
if unchanged:
|
| 172 |
+
require(candidate and candidate["active"] and candidate["payload_sha256"] == payload.payload_sha256
|
| 173 |
+
and candidate["template_id"] == template["id"] and candidate["gpx_sha256"] == payload.sha256
|
| 174 |
+
and candidate["validation_status"] == "passed" and candidate["route_ai_status"] == "ai_pass",
|
| 175 |
+
"incomplete_art_projection", 409)
|
| 176 |
+
result = _receipt(source_id, payload, template, candidate, "unchanged")
|
| 177 |
+
else:
|
| 178 |
+
_put_library(conn, source_id, payload, prepared, payload.payload_sha256, payload.generation, 2)
|
| 179 |
+
candidate = conn.execute("""INSERT INTO art_route_candidates
|
| 180 |
+
(id,template_id,region_code,coordinates,distance_m,validation_status,validation_score,validator_version,
|
| 181 |
+
source,source_id,active,payload_sha256,gpx_sha256,route_ai_status,route_ai_evidence,road_verification,
|
| 182 |
+
graph_sha256,region_name,ai_protocol,accepted_at,coordinate_order,coordinate_crs,sync_generation)
|
| 183 |
+
VALUES (%s,%s,%s,%s,%s,'passed',NULL,%s,%s,%s,true,%s,%s,'ai_pass',%s,%s,%s,%s,%s,%s,'latlon','EPSG:4326',%s)
|
| 184 |
+
ON CONFLICT(source,source_id) DO UPDATE SET template_id=excluded.template_id,region_code=excluded.region_code,
|
| 185 |
+
coordinates=excluded.coordinates,distance_m=excluded.distance_m,validation_status='passed',validation_score=NULL,
|
| 186 |
+
validator_version=excluded.validator_version,active=true,payload_sha256=excluded.payload_sha256,
|
| 187 |
+
gpx_sha256=excluded.gpx_sha256,route_ai_status=excluded.route_ai_status,route_ai_evidence=excluded.route_ai_evidence,
|
| 188 |
+
road_verification=excluded.road_verification,graph_sha256=excluded.graph_sha256,region_name=excluded.region_name,
|
| 189 |
+
ai_protocol=excluded.ai_protocol,accepted_at=excluded.accepted_at,coordinate_order='latlon',coordinate_crs='EPSG:4326',
|
| 190 |
+
sync_generation=excluded.sync_generation,revoked_at=NULL,revocation_reason=NULL,updated_at=now() RETURNING *""",
|
| 191 |
+
(prepared["candidate_id"], template["id"], payload.region_code, Jsonb(prepared["coordinates"]), payload.distance_m,
|
| 192 |
+
payload.validator_version, SOURCE, source_id, payload.payload_sha256, payload.sha256,
|
| 193 |
+
Jsonb(payload.route_ai_evidence), Jsonb(payload.road_verification), payload.graph_sha256,
|
| 194 |
+
payload.region_name, payload.ai_protocol, prepared["accepted_at"], payload.generation)).fetchone()
|
| 195 |
+
_bump(conn)
|
| 196 |
+
result = _receipt(source_id, payload, template, candidate, "updated" if old else "created")
|
| 197 |
+
return result
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _legacy_same(old, payload, prepared):
|
| 201 |
+
return (old["gpx_sha256"] == payload.sha256.lower() and old["label"] == prepared["label"]
|
| 202 |
+
and old["region_code"] == payload.region_code and old["region_name"] == payload.region_name
|
| 203 |
+
and old["accepted_at"] == prepared["accepted_at"] and old["ai_protocol"] == payload.ai_protocol
|
| 204 |
+
and old["collection_finalized"] == payload.collection_finalized)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def store_legacy(conn, source_id, payload, prepared):
|
| 208 |
+
with conn.transaction():
|
| 209 |
+
state = _lock_source(conn)
|
| 210 |
+
old = conn.execute("SELECT * FROM lab_route_library WHERE source=%s AND source_id=%s", (SOURCE, source_id)).fetchone()
|
| 211 |
+
require(not old or old["active"], "explicit_reapproval_required", 409)
|
| 212 |
+
same = bool(old and _legacy_same(old, payload, prepared))
|
| 213 |
+
require(not state["v2_required"] or same, "v2_required", 409)
|
| 214 |
+
if not same:
|
| 215 |
+
content_hash = sha({k: v for k, v in payload.model_dump(mode="json").items() if k != "gpx_base64"})
|
| 216 |
+
_put_library(conn, source_id, payload, prepared, content_hash, 0, 0)
|
| 217 |
+
_bump(conn)
|
| 218 |
+
result = {"source_id": source_id, "label": prepared["label"], "sha256": payload.sha256.lower(), "status": "stored"}
|
| 219 |
+
return result
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _validate_snapshot(payload):
|
| 223 |
+
from app.services.art_route_ingest import HASH
|
| 224 |
+
require(payload.count == len(payload.ids) == len(set(payload.ids)) and all(HASH.fullmatch(i) for i in payload.ids), "invalid_approval_list")
|
| 225 |
+
import hashlib
|
| 226 |
+
require(hashlib.sha256("\n".join(sorted(payload.ids)).encode()).hexdigest() == payload.ids_sha256.lower(), "approval_list_digest_mismatch")
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def _ready(conn, manifest):
|
| 230 |
+
rows = conn.execute("""SELECT l.source_id,l.payload_sha256 AS library_hash,r.payload_sha256 AS route_hash
|
| 231 |
+
FROM lab_route_library l JOIN art_route_candidates r ON r.source=l.source AND r.source_id=l.source_id
|
| 232 |
+
JOIN art_templates t ON t.id=r.template_id
|
| 233 |
+
WHERE l.source=%s AND l.active AND r.active AND l.art_projection_version=2
|
| 234 |
+
AND r.validation_status='passed' AND r.route_ai_status='ai_pass'
|
| 235 |
+
AND r.coordinate_order='latlon' AND r.coordinate_crs='EPSG:4326'
|
| 236 |
+
AND l.gpx_sha256=r.gpx_sha256""", (SOURCE,)).fetchall()
|
| 237 |
+
ready = {r["source_id"] for r in rows if manifest.get(r["source_id"]) == r["library_hash"] == r["route_hash"]}
|
| 238 |
+
require(ready == set(manifest), "incomplete_art_projection", 409)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _deactivate(conn, revoked, generation):
|
| 242 |
+
if revoked:
|
| 243 |
+
conn.execute("""UPDATE lab_route_library SET active=false,revoked_at=now(),revoked_generation=%s,updated_at=now()
|
| 244 |
+
WHERE source=%s AND source_id=ANY(%s)""", (generation, SOURCE, sorted(revoked)))
|
| 245 |
+
conn.execute("""UPDATE art_route_candidates SET active=false,revoked_at=now(),revocation_reason='not_in_approved_snapshot',
|
| 246 |
+
sync_generation=%s,updated_at=now() WHERE source=%s AND source_id=ANY(%s)""", (generation, SOURCE, sorted(revoked)))
|
| 247 |
+
_bump(conn)
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def apply_snapshot(conn, payload):
|
| 251 |
+
_validate_snapshot(payload)
|
| 252 |
+
with conn.transaction():
|
| 253 |
+
state = _lock_source(conn)
|
| 254 |
+
session = None
|
| 255 |
+
if payload.contract_version == 2:
|
| 256 |
+
session = _current_session(conn, state, payload.session_id, payload.generation)
|
| 257 |
+
require(session["mode"] == "full", "partial_session_cannot_revoke", 409)
|
| 258 |
+
require(session["manifest_sha256"] == payload.manifest_sha256 and set(session["manifest"]) == set(payload.ids),
|
| 259 |
+
"snapshot_manifest_mismatch", 409)
|
| 260 |
+
if session["status"] == "completed":
|
| 261 |
+
return session["result"]
|
| 262 |
+
_ready(conn, session["manifest"])
|
| 263 |
+
else:
|
| 264 |
+
require(not state["v2_required"], "v2_required", 409)
|
| 265 |
+
current = {r["source_id"] for r in conn.execute("SELECT source_id FROM lab_route_library WHERE source=%s AND active", (SOURCE,))}
|
| 266 |
+
require(set(payload.ids) <= current, "missing_routes", 409)
|
| 267 |
+
revoked = current - set(payload.ids)
|
| 268 |
+
require(len(revoked) <= max(20, len(current) // 20), "unexpected_deactivations", 409)
|
| 269 |
+
_deactivate(conn, revoked, payload.generation if session else 0)
|
| 270 |
+
if payload.collection_finalized:
|
| 271 |
+
conn.execute("UPDATE lab_route_library SET collection_finalized=true WHERE source=%s AND active AND NOT collection_finalized", (SOURCE,))
|
| 272 |
+
result = {"active_count": payload.count, "deactivated_count": len(revoked)}
|
| 273 |
+
if session:
|
| 274 |
+
result.update(contract_version=2, session_id=str(session["id"]), generation=session["generation"])
|
| 275 |
+
conn.execute("UPDATE lab_route_sync_sessions SET status='completed',result=%s WHERE id=%s", (Jsonb(result), session["id"]))
|
| 276 |
+
conn.execute("UPDATE lab_route_sync_state SET last_full_generation=%s WHERE source=%s", (payload.generation, SOURCE))
|
| 277 |
+
return result
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def complete_partial(conn, session_id, generation):
|
| 281 |
+
with conn.transaction():
|
| 282 |
+
state = _lock_source(conn)
|
| 283 |
+
row = _current_session(conn, state, session_id, generation)
|
| 284 |
+
require(row["mode"] == "partial", "full_session_requires_snapshot", 409)
|
| 285 |
+
_ready(conn, row["manifest"])
|
| 286 |
+
if row["status"] != "completed":
|
| 287 |
+
conn.execute("UPDATE lab_route_sync_sessions SET status='completed' WHERE id=%s", (session_id,))
|
| 288 |
+
result = {**_session_receipt(row), "session_status": "completed", "deactivated_count": 0}
|
| 289 |
+
return result
|
api/ai-service/deployment-source.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"github_sha": "e2b09b366a64790f3dd1dc93870f72061e3c1aab"}
|
api/ai-service/tests/test_route_library_deployment.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deployment must stop before migration if the effective databases differ."""
|
| 2 |
+
import importlib.util
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import pytest
|
| 6 |
+
|
| 7 |
+
ROOT = Path(__file__).resolve().parents[3]
|
| 8 |
+
spec = importlib.util.spec_from_file_location('deployment_preflight', ROOT / 'deploy/huggingface/route-library-preflight.py')
|
| 9 |
+
preflight = importlib.util.module_from_spec(spec)
|
| 10 |
+
spec.loader.exec_module(preflight)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@pytest.fixture
|
| 14 |
+
def configuration(monkeypatch):
|
| 15 |
+
monkeypatch.setenv('DATABASE_URL', 'postgresql://synthetic/auth')
|
| 16 |
+
monkeypatch.setenv('ROUTE_LIBRARY_DATABASE_URL', 'postgresql://synthetic/library')
|
| 17 |
+
monkeypatch.setenv('ART_ROUTE_CATALOG_DATABASE_URL', 'postgresql://synthetic/catalog')
|
| 18 |
+
monkeypatch.delenv('ART_ROUTE_CATALOG_JSON', raising=False)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def observation(identity=('test', 'public', '127.0.0.1', 1), count=3, revision='0013_create_art_routing_tables'):
|
| 22 |
+
return identity, dict.fromkeys(preflight.TABLES, 123), dict.fromkeys(preflight.TABLES, count), [revision]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@pytest.mark.parametrize('failure', ['database', 'schema', 'relation', 'file_catalog'])
|
| 26 |
+
def test_mismatched_placement_never_runs_migration(configuration, monkeypatch, failure):
|
| 27 |
+
values = [observation(), observation(), observation()]
|
| 28 |
+
if failure == 'database': values[1] = observation(identity=('other', 'public', '127.0.0.1', 1))
|
| 29 |
+
if failure == 'schema': values[2] = observation(identity=('test', 'other', '127.0.0.1', 1))
|
| 30 |
+
if failure == 'relation': values[2][1]['art_templates'] = 999
|
| 31 |
+
if failure == 'file_catalog': monkeypatch.setenv('ART_ROUTE_CATALOG_JSON', '/synthetic/catalog.json')
|
| 32 |
+
monkeypatch.setattr(preflight, 'inspect_database', lambda url: values.pop(0))
|
| 33 |
+
calls = []
|
| 34 |
+
with pytest.raises(preflight.DeploymentCheckError): preflight.check_and_migrate(lambda: calls.append(True))
|
| 35 |
+
assert calls == []
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_verified_placement_and_revision_preserve_counts(configuration, monkeypatch):
|
| 39 |
+
values = [observation() for _ in range(3)] + [observation(revision=preflight.REVISION) for _ in range(3)]
|
| 40 |
+
monkeypatch.setattr(preflight, 'inspect_database', lambda url: values.pop(0))
|
| 41 |
+
calls = []
|
| 42 |
+
result = preflight.check_and_migrate(lambda: calls.append(True))
|
| 43 |
+
assert calls == [True] and result['ready'] and result['existing_rows_preserved']
|
| 44 |
+
assert 'synthetic/auth' not in str(result)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@pytest.mark.parametrize('failure', ['revision', 'row_count'])
|
| 48 |
+
def test_failed_post_migration_check_never_claims_ready(configuration, monkeypatch, failure):
|
| 49 |
+
after = observation(revision='wrong' if failure == 'revision' else preflight.REVISION,
|
| 50 |
+
count=4 if failure == 'row_count' else 3)
|
| 51 |
+
values = [observation() for _ in range(3)] + [after for _ in range(3)]
|
| 52 |
+
monkeypatch.setattr(preflight, 'inspect_database', lambda url: values.pop(0))
|
| 53 |
+
with pytest.raises(preflight.DeploymentCheckError): preflight.check_and_migrate(lambda: None)
|
api/ai-service/tests/test_route_library_postgres.py
ADDED
|
@@ -0,0 +1,613 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Synthetic-only integration tests. Never fall back to an application DB URL."""
|
| 2 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 3 |
+
import base64
|
| 4 |
+
import hashlib
|
| 5 |
+
import importlib.util
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import sqlite3
|
| 11 |
+
import subprocess
|
| 12 |
+
import sys
|
| 13 |
+
import urllib.error
|
| 14 |
+
from urllib.parse import urlsplit
|
| 15 |
+
import uuid
|
| 16 |
+
|
| 17 |
+
import pytest
|
| 18 |
+
|
| 19 |
+
SERVICE = Path(__file__).resolve().parents[1]
|
| 20 |
+
ROOT = SERVICE.parents[1]
|
| 21 |
+
sys.path.insert(0, str(SERVICE))
|
| 22 |
+
|
| 23 |
+
from app.api.v1.endpoints import route_library as api
|
| 24 |
+
from app.services import route_library_store as store
|
| 25 |
+
from app.services.art_route_ingest import SOURCE, IngestError, content_sha, sha
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def load_file(name, path):
|
| 29 |
+
spec = importlib.util.spec_from_file_location(name, path)
|
| 30 |
+
module = importlib.util.module_from_spec(spec)
|
| 31 |
+
spec.loader.exec_module(module)
|
| 32 |
+
return module
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@pytest.fixture(scope="module")
|
| 36 |
+
def database():
|
| 37 |
+
dsn = os.environ.get("TEST_ROUTE_LIBRARY_DATABASE_URL")
|
| 38 |
+
if not dsn:
|
| 39 |
+
pytest.skip("Set the dedicated isolated PostgreSQL test URL")
|
| 40 |
+
import psycopg
|
| 41 |
+
from psycopg.conninfo import conninfo_to_dict
|
| 42 |
+
from sqlalchemy import create_engine
|
| 43 |
+
from sqlalchemy.engine import make_url
|
| 44 |
+
from alembic.migration import MigrationContext
|
| 45 |
+
from alembic.operations import Operations
|
| 46 |
+
info = conninfo_to_dict(dsn)
|
| 47 |
+
assert info.get("dbname", "").startswith("route_library_test"), "Refusing a non-test database"
|
| 48 |
+
assert info.get("host", "") in {"localhost", "127.0.0.1", "::1"} or info.get("host", "").startswith("/"), "Refusing a remote database"
|
| 49 |
+
url = make_url(dsn).set(drivername="postgresql+psycopg")
|
| 50 |
+
engine = create_engine(url)
|
| 51 |
+
migration_dir = ROOT / "api/auth-service/alembic/versions"
|
| 52 |
+
original = load_file("art_original_migration", migration_dir / "0013_create_art_routing_tables.py")
|
| 53 |
+
migration = load_file("art_link_migration", migration_dir / "0014_link_lab_art_routes.py")
|
| 54 |
+
with engine.begin() as conn:
|
| 55 |
+
conn.exec_driver_sql("DROP TABLE IF EXISTS lab_route_sync_sessions,lab_route_sync_state,art_route_candidates,art_templates,lab_route_library CASCADE")
|
| 56 |
+
with Operations.context(MigrationContext.configure(conn)):
|
| 57 |
+
original.upgrade()
|
| 58 |
+
migration.upgrade()
|
| 59 |
+
yield dsn, engine, migration
|
| 60 |
+
engine.dispose()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
@pytest.fixture
|
| 64 |
+
def client(database, monkeypatch):
|
| 65 |
+
import psycopg
|
| 66 |
+
from fastapi import FastAPI
|
| 67 |
+
from fastapi.testclient import TestClient
|
| 68 |
+
dsn, _, _ = database
|
| 69 |
+
with psycopg.connect(dsn) as conn:
|
| 70 |
+
conn.execute("TRUNCATE lab_route_sync_sessions,lab_route_sync_state,art_route_candidates,art_templates,lab_route_library CASCADE")
|
| 71 |
+
key = "synthetic-" + uuid.uuid4().hex
|
| 72 |
+
monkeypatch.setenv("ROUTE_LIBRARY_DATABASE_URL", dsn)
|
| 73 |
+
monkeypatch.setenv("ART_ROUTE_CATALOG_DATABASE_URL", dsn)
|
| 74 |
+
monkeypatch.delenv("ART_ROUTE_CATALOG_JSON", raising=False)
|
| 75 |
+
monkeypatch.setenv("ROUTE_LIBRARY_TOKEN", key)
|
| 76 |
+
monkeypatch.setattr(api.settings, "ai_workspace_dir", str(ROOT / "router/ai"))
|
| 77 |
+
api._invalidate_cache()
|
| 78 |
+
app = FastAPI()
|
| 79 |
+
app.include_router(api.router, prefix="/api/v1")
|
| 80 |
+
with TestClient(app) as instance:
|
| 81 |
+
instance.headers["Authorization"] = "Bearer " + key
|
| 82 |
+
yield instance
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
BASE = "/api/v1/ai/internal/route-library"
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def route_id(name):
|
| 89 |
+
return hashlib.sha256(name.encode()).hexdigest()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def test_authenticated_storage_verification_hides_original_coordinates(client, database):
|
| 93 |
+
source_id, packet, _ = fixture_packet('storage-proof')
|
| 94 |
+
session, _ = begin(client, {source_id: packet}, mode='partial')
|
| 95 |
+
receipt = put(client, source_id, packet, session).json()
|
| 96 |
+
response = client.get(BASE + '/routes/' + source_id + '/storage')
|
| 97 |
+
assert response.status_code == 200
|
| 98 |
+
proof = response.json()
|
| 99 |
+
assert proof['verified'] and proof['coordinates_match_gpx'] and all(proof['tables'].values())
|
| 100 |
+
assert proof['art_template_id'] == receipt['art_template_id']
|
| 101 |
+
assert proof['art_route_candidate_id'] == receipt['art_route_candidate_id']
|
| 102 |
+
assert proof['design_ai_status'] == 'ai_reject'
|
| 103 |
+
assert 'coordinates' not in proof and 'gpx' not in proof and 'outline' not in proof
|
| 104 |
+
assert client.get(BASE + '/routes/' + source_id + '/storage', headers={'Authorization': 'Bearer wrong'}).status_code == 401
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def test_deployment_status_requires_live_revision_and_server_preflight(client, database, tmp_path, monkeypatch):
|
| 108 |
+
report = tmp_path / 'report.json'
|
| 109 |
+
provenance = tmp_path / 'source.json'
|
| 110 |
+
monkeypatch.setattr(api, 'DEPLOYMENT_REPORT_PATH', report)
|
| 111 |
+
monkeypatch.setattr(api, 'DEPLOYMENT_SOURCE_PATH', provenance)
|
| 112 |
+
assert client.get(BASE + '/deployment-status').status_code == 503
|
| 113 |
+
report.write_text(json.dumps({'ready': True, 'same_database': True, 'catalog_uses_database': True, 'existing_rows_preserved': True}))
|
| 114 |
+
provenance.write_text(json.dumps({'github_sha': 'a' * 40}))
|
| 115 |
+
with database[1].begin() as connection:
|
| 116 |
+
connection.exec_driver_sql('CREATE TABLE IF NOT EXISTS alembic_version(version_num varchar(128) PRIMARY KEY)')
|
| 117 |
+
connection.exec_driver_sql('DELETE FROM alembic_version')
|
| 118 |
+
connection.exec_driver_sql("INSERT INTO alembic_version VALUES ('0014_link_lab_art_routes')")
|
| 119 |
+
response = client.get(BASE + '/deployment-status')
|
| 120 |
+
assert response.status_code == 200 and response.json()['ready']
|
| 121 |
+
assert response.json()['same_database'] and all(response.json()['tables'].values())
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def fixture_packet(name="one", design_status="ai_reject", design_variant=0):
|
| 125 |
+
points = [[37.5 + .003 * math.sin(i * math.tau / 40), 127.0 + .003 * math.cos(i * math.tau / 40)] for i in range(40)]
|
| 126 |
+
points.append(points[0])
|
| 127 |
+
raw = ('<gpx><trk><trkseg>' + ''.join(f'<trkpt lat="{a}" lon="{b}"/>' for a, b in points) + '</trkseg></trk></gpx>').encode()
|
| 128 |
+
evidence = {"protocol": "synthetic-v1", "status": "ai_pass", "expected_label": "dog", "blind": True,
|
| 129 |
+
"qwen_performed": True, "siglip_support_angles": [0], "siglip_views": [{"top_label": "dog"}],
|
| 130 |
+
"qwen": [{"valid": True, "object": "dog", "clarity": 3}]}
|
| 131 |
+
snapshot = {"design_id": "synthetic_dog", "version": "1", "name": "Synthetic dog",
|
| 132 |
+
"outline": {"coordinate_system": "cartesian_y_up"},
|
| 133 |
+
"points": [[0, 0], [1, 0], [1, 1], [.5 + design_variant * .01, 1.2], [0, 1]]}
|
| 134 |
+
packet = {"contract_version": 2, "label": "dog", "region_code": "synthetic", "region_name": "Synthetic",
|
| 135 |
+
"sha256": hashlib.sha256(raw).hexdigest(), "accepted_at": "2026-11-09T00:00:00+00:00", "review_status": "ai_pass",
|
| 136 |
+
"ai_protocol": "synthetic-v1", "geometry_ok": True, "seoul_ok": True, "collection_finalized": False,
|
| 137 |
+
"gpx_base64": base64.b64encode(raw).decode(),
|
| 138 |
+
"template": {"design_id": snapshot["design_id"], "source_version": "1", "design_digest": sha(snapshot),
|
| 139 |
+
"snapshot": snapshot, "design_ai_status": design_status,
|
| 140 |
+
"design_ai_evidence": evidence if design_status == "ai_pass" else {"status": design_status}},
|
| 141 |
+
"route_ai_evidence": evidence, "road_verification": {"road_only": True, "graph_connectivity_verified": True,
|
| 142 |
+
"access_checked": True, "directions_checked": True, "air_length_m": 0, "field_access_verified": False},
|
| 143 |
+
"graph_sha256": "b" * 64, "distance_m": 2000, "validator_version": "synthetic-v1"}
|
| 144 |
+
return route_id(name), packet, points
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def begin(client, packets, mode="full", request_id=None):
|
| 148 |
+
current = client.get(BASE + "/sync-state").json()
|
| 149 |
+
manifest = {source_id: content_sha(packet) for source_id, packet in packets.items()}
|
| 150 |
+
body = {"client_request_id": request_id or str(uuid.uuid4()), "expected_generation": current["generation"],
|
| 151 |
+
"mode": mode, "manifest": manifest, "manifest_sha256": sha(manifest)}
|
| 152 |
+
response = client.post(BASE + "/sync-sessions", json=body)
|
| 153 |
+
assert response.status_code == 200, "Session start failed"
|
| 154 |
+
return response.json(), body
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def put(client, source_id, packet, session, **actions):
|
| 158 |
+
body = {**packet, "session_id": session["session_id"], "generation": session["generation"],
|
| 159 |
+
"payload_sha256": content_sha(packet), **actions}
|
| 160 |
+
return client.put(BASE + "/routes/" + source_id, json=body)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def finish(client, session, packets):
|
| 164 |
+
ids = sorted(packets)
|
| 165 |
+
return client.post(BASE + "/approval-snapshot", json={"contract_version": 2, "session_id": session["session_id"],
|
| 166 |
+
"generation": session["generation"], "manifest_sha256": session["manifest_sha256"], "ids": ids, "count": len(ids),
|
| 167 |
+
"ids_sha256": hashlib.sha256("\n".join(ids).encode()).hexdigest(), "collection_finalized": False})
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def query(database, sql, parameters=()):
|
| 171 |
+
import psycopg
|
| 172 |
+
from psycopg.rows import dict_row
|
| 173 |
+
with psycopg.connect(database[0], row_factory=dict_row) as conn:
|
| 174 |
+
return conn.execute(sql, parameters).fetchall()
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def test_atomic_ids_gpx_and_independent_design_review(client, database):
|
| 178 |
+
source_id, packet, points = fixture_packet()
|
| 179 |
+
session, _ = begin(client, {source_id: packet})
|
| 180 |
+
response = put(client, source_id, packet, session)
|
| 181 |
+
assert response.status_code == 200
|
| 182 |
+
receipt = response.json()
|
| 183 |
+
assert receipt["design_ai_status"] == "ai_reject"
|
| 184 |
+
assert receipt["route_validation_status"] == "passed"
|
| 185 |
+
template = query(database, "SELECT * FROM art_templates")[0]
|
| 186 |
+
candidate = query(database, "SELECT * FROM art_route_candidates")[0]
|
| 187 |
+
library = query(database, "SELECT * FROM lab_route_library")[0]
|
| 188 |
+
assert str(template["id"]) == receipt["art_template_id"]
|
| 189 |
+
assert str(candidate["id"]) == receipt["art_route_candidate_id"]
|
| 190 |
+
assert candidate["template_id"] == template["id"]
|
| 191 |
+
assert candidate["coordinates"] == points
|
| 192 |
+
assert candidate["coordinate_crs"] == "EPSG:4326" and candidate["coordinate_order"] == "latlon"
|
| 193 |
+
assert candidate["coordinates"] != library["outline"]
|
| 194 |
+
assert template["design_ai_status"] == "ai_reject" and candidate["route_ai_status"] == "ai_pass"
|
| 195 |
+
assert finish(client, session, {source_id: packet}).status_code == 200
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def test_retransmission_is_a_real_noop_and_receipts_survive_completion(client, database):
|
| 199 |
+
source_id, packet, _ = fixture_packet()
|
| 200 |
+
session, _ = begin(client, {source_id: packet})
|
| 201 |
+
first = put(client, source_id, packet, session).json()
|
| 202 |
+
before = query(database, "SELECT updated_at FROM lab_route_library")[0]
|
| 203 |
+
second = put(client, source_id, packet, session).json()
|
| 204 |
+
assert second["result"] == "unchanged"
|
| 205 |
+
assert first["art_template_id"] == second["art_template_id"]
|
| 206 |
+
assert first["art_route_candidate_id"] == second["art_route_candidate_id"]
|
| 207 |
+
assert query(database, "SELECT updated_at FROM lab_route_library")[0] == before
|
| 208 |
+
assert finish(client, session, {source_id: packet}).status_code == 200
|
| 209 |
+
assert put(client, source_id, packet, session).json()["result"] == "unchanged"
|
| 210 |
+
assert finish(client, session, {source_id: packet}).json()["deactivated_count"] == 0
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def test_concurrent_same_route_has_one_identity(client, database):
|
| 214 |
+
source_id, packet, _ = fixture_packet()
|
| 215 |
+
session, _ = begin(client, {source_id: packet})
|
| 216 |
+
with ThreadPoolExecutor(max_workers=2) as pool:
|
| 217 |
+
responses = list(pool.map(lambda _: put(client, source_id, packet, session), range(2)))
|
| 218 |
+
assert [r.status_code for r in responses] == [200, 200]
|
| 219 |
+
assert len({r.json()["art_route_candidate_id"] for r in responses}) == 1
|
| 220 |
+
for table in ["art_templates", "lab_route_library", "art_route_candidates"]:
|
| 221 |
+
assert query(database, "SELECT count(*) AS n FROM " + table)[0]["n"] == 1
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
@pytest.mark.parametrize("function", ["_template", "_put_library", "_bump"])
|
| 225 |
+
def test_failures_rollback_all_three_tables(client, database, monkeypatch, function):
|
| 226 |
+
source_id, packet, _ = fixture_packet()
|
| 227 |
+
session, _ = begin(client, {source_id: packet})
|
| 228 |
+
original = getattr(store, function)
|
| 229 |
+
def fail(*args, **kwargs):
|
| 230 |
+
original(*args, **kwargs)
|
| 231 |
+
raise IngestError("synthetic_failure", 503)
|
| 232 |
+
monkeypatch.setattr(store, function, fail)
|
| 233 |
+
assert put(client, source_id, packet, session).status_code == 503
|
| 234 |
+
for table in ["art_templates", "lab_route_library", "art_route_candidates"]:
|
| 235 |
+
assert query(database, "SELECT count(*) AS n FROM " + table)[0]["n"] == 0
|
| 236 |
+
assert query(database, "SELECT catalog_revision FROM lab_route_sync_state")[0]["catalog_revision"] == 0
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def test_changed_route_requires_compare_and_swap(client, database):
|
| 240 |
+
source_id, packet, _ = fixture_packet()
|
| 241 |
+
session, _ = begin(client, {source_id: packet})
|
| 242 |
+
first = put(client, source_id, packet, session).json()
|
| 243 |
+
changed = {**packet, "distance_m": packet["distance_m"] + 1}
|
| 244 |
+
newer, _ = begin(client, {source_id: changed})
|
| 245 |
+
assert put(client, source_id, changed, newer).status_code == 409
|
| 246 |
+
assert query(database, "SELECT distance_m FROM art_route_candidates")[0]["distance_m"] == packet["distance_m"]
|
| 247 |
+
response = put(client, source_id, changed, newer, previous_payload_sha256=content_sha(packet))
|
| 248 |
+
assert response.status_code == 200
|
| 249 |
+
assert response.json()["art_route_candidate_id"] == first["art_route_candidate_id"]
|
| 250 |
+
assert response.json()["result"] == "updated"
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def test_same_source_design_version_different_digest_is_not_overwritten(client, database):
|
| 254 |
+
a, one, _ = fixture_packet("one")
|
| 255 |
+
b, two, _ = fixture_packet("two", design_variant=1)
|
| 256 |
+
session, _ = begin(client, {a: one, b: two})
|
| 257 |
+
assert put(client, a, one, session).status_code == put(client, b, two, session).status_code == 200
|
| 258 |
+
templates = query(database, "SELECT id,version,source_design_version FROM art_templates")
|
| 259 |
+
assert len(templates) == 2 and len({t["version"] for t in templates}) == 2
|
| 260 |
+
assert all(t["source_design_version"] == "1" for t in templates)
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def test_revocation_fences_old_requests_and_requires_explicit_new_approval(client, database):
|
| 264 |
+
a, one, _ = fixture_packet("one")
|
| 265 |
+
b, two, _ = fixture_packet("two")
|
| 266 |
+
session, _ = begin(client, {a: one, b: two})
|
| 267 |
+
assert put(client, a, one, session).status_code == put(client, b, two, session).status_code == 200
|
| 268 |
+
assert finish(client, session, {a: one, b: two}).status_code == 200
|
| 269 |
+
newer, _ = begin(client, {a: one})
|
| 270 |
+
assert finish(client, newer, {a: one}).json()["deactivated_count"] == 1
|
| 271 |
+
assert not query(database, "SELECT active FROM lab_route_library WHERE source_id=%s", (b,))[0]["active"]
|
| 272 |
+
assert not query(database, "SELECT active FROM art_route_candidates WHERE source_id=%s", (b,))[0]["active"]
|
| 273 |
+
assert query(database, "SELECT count(*) AS n FROM art_templates")[0]["n"] == 1
|
| 274 |
+
assert put(client, b, two, session).status_code == 409
|
| 275 |
+
retry, _ = begin(client, {b: two}, "partial")
|
| 276 |
+
assert put(client, b, two, retry).status_code == 409
|
| 277 |
+
assert put(client, b, two, retry, reapprove=True).status_code == 409
|
| 278 |
+
reapproved = {**two, "accepted_at": "2026-11-10T00:00:00+00:00"}
|
| 279 |
+
latest, _ = begin(client, {b: reapproved}, "partial")
|
| 280 |
+
assert put(client, b, reapproved, latest, reapprove=True, previous_payload_sha256=content_sha(two)).status_code == 200
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def test_incomplete_and_partial_snapshots_cannot_revoke(client, database):
|
| 284 |
+
a, one, _ = fixture_packet("one")
|
| 285 |
+
b, two, _ = fixture_packet("two")
|
| 286 |
+
session, _ = begin(client, {a: one, b: two})
|
| 287 |
+
assert put(client, a, one, session).status_code == 200
|
| 288 |
+
assert finish(client, session, {a: one, b: two}).status_code == 409
|
| 289 |
+
partial, _ = begin(client, {b: two}, "partial")
|
| 290 |
+
assert put(client, b, two, partial).status_code == 200
|
| 291 |
+
assert finish(client, partial, {b: two}).status_code == 409
|
| 292 |
+
complete = client.post(BASE + "/sync-sessions/" + partial["session_id"] + "/complete", params={"generation": partial["generation"]})
|
| 293 |
+
assert complete.status_code == 200 and complete.json()["deactivated_count"] == 0
|
| 294 |
+
assert query(database, "SELECT count(*) AS n FROM lab_route_library WHERE active")[0]["n"] == 2
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def test_session_begin_is_idempotent_and_cannot_replay_after_new_generation(client):
|
| 298 |
+
source_id, packet, _ = fixture_packet()
|
| 299 |
+
session, body = begin(client, {source_id: packet})
|
| 300 |
+
assert client.post(BASE + "/sync-sessions", json=body).json() == session
|
| 301 |
+
newer, _ = begin(client, {source_id: packet})
|
| 302 |
+
assert newer["generation"] == session["generation"] + 1
|
| 303 |
+
assert client.post(BASE + "/sync-sessions", json=body).status_code == 409
|
| 304 |
+
assert put(client, source_id, packet, session).status_code == 409
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def test_legacy_request_compatibility_and_v2_write_fence(client, database):
|
| 308 |
+
source_id, packet, _ = fixture_packet()
|
| 309 |
+
keys = {"label", "region_code", "region_name", "sha256", "accepted_at", "review_status", "ai_protocol", "geometry_ok", "seoul_ok", "collection_finalized", "gpx_base64"}
|
| 310 |
+
legacy = {k: v for k, v in packet.items() if k in keys}
|
| 311 |
+
assert client.put(BASE + "/routes/" + source_id, json=legacy).status_code == 200
|
| 312 |
+
assert query(database, "SELECT count(*) AS n FROM art_templates")[0]["n"] == 0
|
| 313 |
+
session, _ = begin(client, {source_id: packet})
|
| 314 |
+
assert put(client, source_id, packet, session).status_code == 200
|
| 315 |
+
assert client.put(BASE + "/routes/" + source_id, json=legacy).status_code == 200
|
| 316 |
+
assert client.put(BASE + "/routes/" + source_id, json={**legacy, "region_name": "changed"}).status_code == 409
|
| 317 |
+
assert client.post(BASE + "/approval-snapshot", json={"ids": [], "count": 0, "ids_sha256": hashlib.sha256(b"").hexdigest()}).status_code == 409
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def test_v2_rejects_client_coordinates_and_redacts_validation_values(client):
|
| 321 |
+
source_id, packet, _ = fixture_packet()
|
| 322 |
+
session, _ = begin(client, {source_id: packet})
|
| 323 |
+
response = put(client, source_id, {**packet, "coordinates": "private-input-marker"}, session)
|
| 324 |
+
assert response.status_code == 422
|
| 325 |
+
assert "private-input-marker" not in response.text
|
| 326 |
+
assert packet["gpx_base64"] not in response.text
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def test_catalog_and_live_outline_cache_drop_revoked_routes(client, database, monkeypatch):
|
| 330 |
+
from app.services.art_route_catalog import load_art_route_catalog
|
| 331 |
+
source_id, packet, _ = fixture_packet()
|
| 332 |
+
session, _ = begin(client, {source_id: packet})
|
| 333 |
+
assert put(client, source_id, packet, session).status_code == 200
|
| 334 |
+
monkeypatch.setenv("ART_ROUTE_COORDINATE_ORDER", "lonlat")
|
| 335 |
+
assert "dog" in load_art_route_catalog() # v2 order is per-row, independent of legacy config.
|
| 336 |
+
assert "dog" in api.active_outlines()
|
| 337 |
+
empty, _ = begin(client, {})
|
| 338 |
+
assert finish(client, empty, {}).status_code == 200
|
| 339 |
+
assert load_art_route_catalog() == {} and api.active_outlines() == {}
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def test_database_outage_never_replays_live_outline_cache(client, monkeypatch):
|
| 343 |
+
source_id, packet, _ = fixture_packet()
|
| 344 |
+
session, _ = begin(client, {source_id: packet})
|
| 345 |
+
assert put(client, source_id, packet, session).status_code == 200
|
| 346 |
+
assert api.active_outlines()
|
| 347 |
+
monkeypatch.delenv("ROUTE_LIBRARY_DATABASE_URL")
|
| 348 |
+
assert api.active_outlines() == {}
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def make_lab(tmp_path, name="sender"):
|
| 352 |
+
source_id, packet, points = fixture_packet(name)
|
| 353 |
+
root = tmp_path / "lab"
|
| 354 |
+
(root / "gpx/dog").mkdir(parents=True)
|
| 355 |
+
(root / "designs/dog").mkdir(parents=True)
|
| 356 |
+
design_path = root / "designs/dog/synthetic_dog.json"
|
| 357 |
+
design_path.write_text(json.dumps(packet["template"]["snapshot"]))
|
| 358 |
+
gpx_path = root / "gpx/dog" / (source_id + ".gpx")
|
| 359 |
+
gpx_path.write_bytes(base64.b64decode(packet["gpx_base64"]))
|
| 360 |
+
record_path = root / "synthetic_record.json"
|
| 361 |
+
record_path.write_text(json.dumps({"metadata": {"design_file": str(design_path), "design_digest": packet["template"]["design_digest"],
|
| 362 |
+
"graph_sha256": packet["graph_sha256"]}, "assessment": {"success": True, "route_valid": True},
|
| 363 |
+
"route": {"design_id": "synthetic_dog", "design_version": "1", "length_m": 2000,
|
| 364 |
+
"verification": packet["road_verification"], "gps": [[b, a] for a, b in points]}}))
|
| 365 |
+
with sqlite3.connect(root / "inventory.sqlite3") as db:
|
| 366 |
+
db.executescript("""CREATE TABLE items(id TEXT PRIMARY KEY,kind TEXT,label TEXT,city_id TEXT,city_name TEXT,
|
| 367 |
+
design_id TEXT,record_path TEXT,ai_json TEXT,geometry_ok INT,seoul_ok INT,status TEXT);
|
| 368 |
+
CREATE TABLE accepted(item_id TEXT PRIMARY KEY,label TEXT,city_id TEXT,design_id TEXT,accepted_at TEXT);
|
| 369 |
+
CREATE TABLE approval_revocations(item_id TEXT PRIMARY KEY);""")
|
| 370 |
+
db.execute("INSERT INTO items VALUES (?,?,?,?,?,?,?,?,?,?,?)", (source_id, "route", "dog", "synthetic", "Synthetic",
|
| 371 |
+
"synthetic_dog", str(record_path), json.dumps(packet["route_ai_evidence"]), 1, 1, "ai_pass"))
|
| 372 |
+
design_item_id = hashlib.sha256(("design:dog:" + packet["template"]["design_digest"]).encode()).hexdigest()
|
| 373 |
+
db.execute("INSERT INTO items(id,kind,status,ai_json) VALUES (?,?,?,?)", (design_item_id, "design", "ai_reject", json.dumps({"status": "ai_reject"})))
|
| 374 |
+
db.execute("INSERT INTO accepted VALUES (?,?,?,?,?)", (source_id, "dog", "synthetic", "synthetic_dog", packet["accepted_at"]))
|
| 375 |
+
return root, source_id
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def test_http_sender_resumes_a_lost_success_without_duplicate_rows(client, database, tmp_path, monkeypatch):
|
| 379 |
+
sender = load_file("art_http_sender", ROOT / "tools/sync_lab_gpx.py")
|
| 380 |
+
root, source_id = make_lab(tmp_path)
|
| 381 |
+
lost, sessions, puts = [], [], []
|
| 382 |
+
def http(url, token, method, payload):
|
| 383 |
+
parsed = urlsplit(url)
|
| 384 |
+
response = client.request(method, parsed.path + ("?" + parsed.query if parsed.query else ""), json=payload)
|
| 385 |
+
assert response.status_code == 200, "Synthetic HTTP request failed"
|
| 386 |
+
if parsed.path.endswith("/sync-sessions"):
|
| 387 |
+
sessions.append(response.json()["session_id"])
|
| 388 |
+
if method == "PUT":
|
| 389 |
+
puts.append(response.json()["art_route_candidate_id"])
|
| 390 |
+
if not lost:
|
| 391 |
+
lost.append(True)
|
| 392 |
+
raise urllib.error.URLError("synthetic response loss")
|
| 393 |
+
return response.json()
|
| 394 |
+
monkeypatch.setattr(sender, "send", http)
|
| 395 |
+
state = tmp_path / "sender/state.sqlite3"
|
| 396 |
+
with pytest.raises(urllib.error.URLError):
|
| 397 |
+
sender.sync_v2(root, "https://example.test" + BASE, "synthetic", state, probe_id=source_id)
|
| 398 |
+
result = sender.sync_v2(root, "https://example.test" + BASE, "synthetic", state, probe_id=source_id)
|
| 399 |
+
assert result["snapshot_sent"] is False and result["deactivated_count"] == 0
|
| 400 |
+
assert len(set(sessions)) == len(set(puts)) == 1
|
| 401 |
+
assert query(database, "SELECT count(*) AS n FROM art_route_candidates")[0]["n"] == 1
|
| 402 |
+
with sqlite3.connect(state) as db:
|
| 403 |
+
saved = db.execute("SELECT template_id,candidate_id FROM http_v2_sent").fetchone()
|
| 404 |
+
assert saved and saved[1] == puts[0]
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def test_sender_does_not_mark_legacy_success_as_art_storage(client, tmp_path, monkeypatch):
|
| 408 |
+
sender = load_file("art_http_sender_bad_ack", ROOT / "tools/sync_lab_gpx.py")
|
| 409 |
+
root, source_id = make_lab(tmp_path)
|
| 410 |
+
def http(url, token, method, payload):
|
| 411 |
+
response = client.request(method, urlsplit(url).path, json=payload)
|
| 412 |
+
assert response.status_code == 200
|
| 413 |
+
result = response.json()
|
| 414 |
+
if method == "PUT":
|
| 415 |
+
result.pop("art_template_id")
|
| 416 |
+
return result
|
| 417 |
+
monkeypatch.setattr(sender, "send", http)
|
| 418 |
+
state = tmp_path / "sender/state.sqlite3"
|
| 419 |
+
with pytest.raises(RuntimeError, match="receipt"):
|
| 420 |
+
sender.sync_v2(root, "https://example.test" + BASE, "synthetic", state, probe_id=source_id)
|
| 421 |
+
with sqlite3.connect(state) as db:
|
| 422 |
+
assert db.execute("SELECT count(*) FROM http_v2_sent").fetchone()[0] == 0
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def test_migration_preserves_legacy_rows_and_downgrade_keeps_library(client, database):
|
| 426 |
+
from alembic.migration import MigrationContext
|
| 427 |
+
from alembic.operations import Operations
|
| 428 |
+
_, engine, migration = database
|
| 429 |
+
with engine.begin() as conn:
|
| 430 |
+
with Operations.context(MigrationContext.configure(conn)):
|
| 431 |
+
migration.downgrade()
|
| 432 |
+
old_id = str(uuid.uuid4())
|
| 433 |
+
conn.exec_driver_sql("INSERT INTO art_templates(id,template_key,name,version,svg) VALUES (%s,'legacy','legacy',1,'<svg/>')", (old_id,))
|
| 434 |
+
conn.exec_driver_sql("""INSERT INTO lab_route_library(source,source_id,label,region_code,region_name,gpx_sha256,
|
| 435 |
+
accepted_at,ai_protocol,gpx,outline) VALUES ('legacy','legacy','dog','legacy','legacy','legacy',now(),'legacy',%s,'[]')""", (b"synthetic-evidence",))
|
| 436 |
+
with Operations.context(MigrationContext.configure(conn)):
|
| 437 |
+
migration.upgrade()
|
| 438 |
+
assert conn.exec_driver_sql("SELECT design_ai_status FROM art_templates WHERE id=%s", (old_id,)).scalar() == "unknown"
|
| 439 |
+
assert conn.exec_driver_sql("SELECT gpx FROM lab_route_library WHERE source='legacy'").scalar() == b"synthetic-evidence"
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
def test_snapshot_failure_rolls_back_both_deactivations(client, database, monkeypatch):
|
| 443 |
+
source_id, packet, _ = fixture_packet()
|
| 444 |
+
session, _ = begin(client, {source_id: packet})
|
| 445 |
+
assert put(client, source_id, packet, session).status_code == 200
|
| 446 |
+
empty, _ = begin(client, {})
|
| 447 |
+
original = store._deactivate
|
| 448 |
+
def fail(*args):
|
| 449 |
+
original(*args)
|
| 450 |
+
raise IngestError("synthetic_snapshot_failure", 503)
|
| 451 |
+
monkeypatch.setattr(store, "_deactivate", fail)
|
| 452 |
+
assert finish(client, empty, {}).status_code == 503
|
| 453 |
+
assert query(database, "SELECT active FROM lab_route_library")[0]["active"]
|
| 454 |
+
assert query(database, "SELECT active FROM art_route_candidates")[0]["active"]
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def test_cannot_claim_approved_design_without_its_own_review(client):
|
| 458 |
+
source_id, packet, _ = fixture_packet(design_status="ai_pass")
|
| 459 |
+
packet["template"]["design_ai_evidence"] = {"status": "ai_pass"}
|
| 460 |
+
session, _ = begin(client, {source_id: packet})
|
| 461 |
+
assert put(client, source_id, packet, session).status_code == 422
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def test_sender_rejects_gpx_that_does_not_match_source_gps(tmp_path):
|
| 465 |
+
sender = load_file("art_sender_source_gpx", ROOT / "tools/sync_lab_gpx.py")
|
| 466 |
+
root, _ = make_lab(tmp_path)
|
| 467 |
+
record_path = root / "synthetic_record.json"
|
| 468 |
+
record = json.loads(record_path.read_text())
|
| 469 |
+
record["route"]["gps"][0][0] += .1
|
| 470 |
+
record_path.write_text(json.dumps(record))
|
| 471 |
+
with pytest.raises(RuntimeError, match="approved source GPS"):
|
| 472 |
+
sender.art_packets(root)
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def test_sender_source_change_prevents_snapshot(client, tmp_path, monkeypatch):
|
| 476 |
+
sender = load_file("art_sender_source_change", ROOT / "tools/sync_lab_gpx.py")
|
| 477 |
+
root, _ = make_lab(tmp_path)
|
| 478 |
+
snapshot_calls = []
|
| 479 |
+
def http(url, token, method, payload):
|
| 480 |
+
path = urlsplit(url).path
|
| 481 |
+
if path.endswith("approval-snapshot"):
|
| 482 |
+
snapshot_calls.append(True)
|
| 483 |
+
response = client.request(method, path, json=payload)
|
| 484 |
+
assert response.status_code == 200
|
| 485 |
+
if method == "PUT":
|
| 486 |
+
with sqlite3.connect(root / "inventory.sqlite3") as db:
|
| 487 |
+
db.execute("UPDATE items SET status='geometry_rejected' WHERE kind='route'")
|
| 488 |
+
return response.json()
|
| 489 |
+
monkeypatch.setattr(sender, "send", http)
|
| 490 |
+
with pytest.raises(RuntimeError):
|
| 491 |
+
sender.sync_v2(root, "https://example.test" + BASE, "synthetic", tmp_path / "state/db.sqlite3")
|
| 492 |
+
assert not snapshot_calls
|
| 493 |
+
|
| 494 |
+
|
| 495 |
+
def test_full_alembic_history_with_synthetic_legacy_prerequisites(database):
|
| 496 |
+
"""The repository's old history assumes these two base domain tables exist."""
|
| 497 |
+
import psycopg
|
| 498 |
+
from psycopg import sql
|
| 499 |
+
from sqlalchemy.engine import make_url
|
| 500 |
+
database_name = "route_library_test_history_" + uuid.uuid4().hex[:8]
|
| 501 |
+
test_url = make_url(database[0]).set(database=database_name)
|
| 502 |
+
with psycopg.connect(database[0], autocommit=True) as admin:
|
| 503 |
+
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database_name)))
|
| 504 |
+
try:
|
| 505 |
+
with psycopg.connect(test_url.render_as_string(hide_password=False)) as conn:
|
| 506 |
+
conn.execute("CREATE TABLE run_sessions(id bigint PRIMARY KEY)")
|
| 507 |
+
conn.execute("CREATE TABLE routes(id bigint PRIMARY KEY)")
|
| 508 |
+
environment = {**os.environ, "DATABASE_URL": test_url.set(drivername="postgresql+psycopg").render_as_string(hide_password=False)}
|
| 509 |
+
auth = ROOT / "api/auth-service"
|
| 510 |
+
def migrate(*arguments):
|
| 511 |
+
result = subprocess.run([sys.executable, "-m", "alembic", *arguments], cwd=auth,
|
| 512 |
+
env=environment, capture_output=True, text=True)
|
| 513 |
+
assert result.returncode == 0, "Isolated full-history migration failed"
|
| 514 |
+
migrate("upgrade", "head")
|
| 515 |
+
with psycopg.connect(test_url.render_as_string(hide_password=False)) as conn:
|
| 516 |
+
assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == "0014_link_lab_art_routes"
|
| 517 |
+
conn.execute("ALTER TABLE alembic_version ALTER COLUMN version_num TYPE varchar(32)")
|
| 518 |
+
migrate("upgrade", "head") # Widen an existing version table without replacing its revision.
|
| 519 |
+
migrate("downgrade", "0013_create_art_routing_tables")
|
| 520 |
+
migrate("upgrade", "head")
|
| 521 |
+
with psycopg.connect(test_url.render_as_string(hide_password=False)) as conn:
|
| 522 |
+
assert conn.execute("SELECT character_maximum_length FROM information_schema.columns WHERE table_name='alembic_version' AND column_name='version_num'").fetchone()[0] == 128
|
| 523 |
+
finally:
|
| 524 |
+
with psycopg.connect(database[0], autocommit=True) as admin:
|
| 525 |
+
admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database_name)))
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def test_commit_time_failure_never_returns_storage_ids(client, database):
|
| 529 |
+
import psycopg
|
| 530 |
+
source_id, packet, _ = fixture_packet()
|
| 531 |
+
session, _ = begin(client, {source_id: packet})
|
| 532 |
+
with psycopg.connect(database[0]) as conn:
|
| 533 |
+
conn.execute("""CREATE FUNCTION synthetic_fail_art_commit() RETURNS trigger LANGUAGE plpgsql AS $$
|
| 534 |
+
BEGIN RAISE EXCEPTION 'synthetic_deferred_failure'; END; $$""")
|
| 535 |
+
conn.execute("""CREATE CONSTRAINT TRIGGER synthetic_art_commit AFTER INSERT ON art_route_candidates
|
| 536 |
+
DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION synthetic_fail_art_commit()""")
|
| 537 |
+
try:
|
| 538 |
+
response = put(client, source_id, packet, session)
|
| 539 |
+
assert response.status_code == 503 and "art_template_id" not in response.json()
|
| 540 |
+
for table in ["art_templates", "lab_route_library", "art_route_candidates"]:
|
| 541 |
+
assert query(database, "SELECT count(*) AS n FROM " + table)[0]["n"] == 0
|
| 542 |
+
finally:
|
| 543 |
+
with psycopg.connect(database[0]) as conn:
|
| 544 |
+
conn.execute("DROP TRIGGER synthetic_art_commit ON art_route_candidates")
|
| 545 |
+
conn.execute("DROP FUNCTION synthetic_fail_art_commit()")
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
def test_generation_does_not_return_a_catalog_revoked_while_computing(client, monkeypatch):
|
| 549 |
+
from app.api.v1.endpoints import route_generation
|
| 550 |
+
from fastapi import HTTPException
|
| 551 |
+
from types import SimpleNamespace
|
| 552 |
+
source_id, packet, _ = fixture_packet()
|
| 553 |
+
session, _ = begin(client, {source_id: packet})
|
| 554 |
+
assert put(client, source_id, packet, session).status_code == 200
|
| 555 |
+
def compute(_):
|
| 556 |
+
empty, _ = begin(client, {})
|
| 557 |
+
assert finish(client, empty, {}).status_code == 200
|
| 558 |
+
return {"routes": []}
|
| 559 |
+
monkeypatch.setattr(route_generation, "_generate_routes", compute)
|
| 560 |
+
monkeypatch.setattr(route_generation, "_load_pipeline_module", lambda _: SimpleNamespace())
|
| 561 |
+
with pytest.raises(HTTPException) as failure:
|
| 562 |
+
route_generation.generate_routes(route_generation.RouteGenerateRequest(source="current", latitude=37, longitude=127))
|
| 563 |
+
assert failure.value.status_code == 409 and failure.value.detail == "route_library_changed_retry"
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
def test_snapshot_cannot_confirm_a_failed_candidate(client, database):
|
| 567 |
+
import psycopg
|
| 568 |
+
source_id, packet, _ = fixture_packet()
|
| 569 |
+
session, _ = begin(client, {source_id: packet})
|
| 570 |
+
assert put(client, source_id, packet, session).status_code == 200
|
| 571 |
+
with psycopg.connect(database[0]) as conn:
|
| 572 |
+
conn.execute("UPDATE art_route_candidates SET validation_status='failed'")
|
| 573 |
+
assert finish(client, session, {source_id: packet}).status_code == 409
|
| 574 |
+
assert put(client, source_id, packet, session).status_code == 409
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def test_nonfinite_snapshot_values_are_a_redacted_client_error(client):
|
| 578 |
+
source_id, packet, _ = fixture_packet()
|
| 579 |
+
session, _ = begin(client, {source_id: packet})
|
| 580 |
+
body = {**packet, "session_id": session["session_id"], "generation": session["generation"], "payload_sha256": content_sha(packet)}
|
| 581 |
+
body["template"]["snapshot"]["points"][0][0] = float("nan")
|
| 582 |
+
response = client.put(BASE + "/routes/" + source_id, content=json.dumps(body), headers={"Content-Type": "application/json"})
|
| 583 |
+
assert response.status_code == 422 and response.json()["detail"] == "invalid_json_value"
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def test_sender_full_http_sync_rechecks_receipts_without_business_updates(client, database, tmp_path, monkeypatch):
|
| 587 |
+
sender = load_file("art_sender_complete_http", ROOT / "tools/sync_lab_gpx.py")
|
| 588 |
+
root, _ = make_lab(tmp_path)
|
| 589 |
+
def http(url, token, method, payload):
|
| 590 |
+
response = client.request(method, urlsplit(url).path, json=payload)
|
| 591 |
+
assert response.status_code == 200, "Synthetic full sync failed"
|
| 592 |
+
return response.json()
|
| 593 |
+
monkeypatch.setattr(sender, "send", http)
|
| 594 |
+
state = tmp_path / "sender/state.sqlite3"
|
| 595 |
+
first = sender.sync_v2(root, "https://example.test" + BASE, "synthetic", state)
|
| 596 |
+
timestamp = query(database, "SELECT updated_at FROM art_route_candidates")[0]
|
| 597 |
+
second = sender.sync_v2(root, "https://example.test" + BASE, "synthetic", state)
|
| 598 |
+
assert first["snapshot_sent"] and second["snapshot_sent"]
|
| 599 |
+
assert second["generation"] == first["generation"] + 1
|
| 600 |
+
assert query(database, "SELECT updated_at FROM art_route_candidates")[0] == timestamp
|
| 601 |
+
assert query(database, "SELECT count(*) AS n FROM art_route_candidates")[0]["n"] == 1
|
| 602 |
+
|
| 603 |
+
|
| 604 |
+
def test_sender_check_only_never_reads_auth_file_or_calls_http(tmp_path, monkeypatch, capsys):
|
| 605 |
+
sender = load_file("art_sender_check_only", ROOT / "tools/sync_lab_gpx.py")
|
| 606 |
+
root, _ = make_lab(tmp_path)
|
| 607 |
+
def forbidden(*args, **kwargs):
|
| 608 |
+
raise AssertionError("check-only attempted HTTP")
|
| 609 |
+
monkeypatch.setattr(sender, "send", forbidden)
|
| 610 |
+
monkeypatch.setattr(sys, "argv", ["sync_lab_gpx.py", "--root", str(root), "--url", "https://example.test",
|
| 611 |
+
"--token-file", str(tmp_path / "auth-file-does-not-exist"), "--contract-version", "2", "--check-only"])
|
| 612 |
+
sender.main()
|
| 613 |
+
assert json.loads(capsys.readouterr().out)["http_requests"] == 0
|
api/auth-service/alembic/env.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
| 1 |
from logging.config import fileConfig
|
| 2 |
|
| 3 |
from alembic import context
|
| 4 |
-
from sqlalchemy import engine_from_config, pool
|
|
|
|
| 5 |
|
| 6 |
from app.core.config import settings
|
| 7 |
from app.db.base import Base
|
|
@@ -14,6 +15,16 @@ if config.config_file_name is not None:
|
|
| 14 |
target_metadata = Base.metadata
|
| 15 |
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
def get_url() -> str:
|
| 18 |
return settings.database_url
|
| 19 |
|
|
@@ -41,13 +52,14 @@ def run_migrations_online() -> None:
|
|
| 41 |
poolclass=pool.NullPool,
|
| 42 |
)
|
| 43 |
|
| 44 |
-
with connectable.connect() as connection:
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
|
|
|
| 51 |
with context.begin_transaction():
|
| 52 |
context.run_migrations()
|
| 53 |
|
|
|
|
| 1 |
from logging.config import fileConfig
|
| 2 |
|
| 3 |
from alembic import context
|
| 4 |
+
from sqlalchemy import engine_from_config, pool, inspect, text, String
|
| 5 |
+
from alembic.ddl.postgresql import PostgresqlImpl
|
| 6 |
|
| 7 |
from app.core.config import settings
|
| 8 |
from app.db.base import Base
|
|
|
|
| 15 |
target_metadata = Base.metadata
|
| 16 |
|
| 17 |
|
| 18 |
+
class LongRevisionPostgresqlImpl(PostgresqlImpl):
|
| 19 |
+
"""Existing revision names exceed Alembic's default 32 characters."""
|
| 20 |
+
__dialect__ = "postgresql"
|
| 21 |
+
|
| 22 |
+
def version_table_impl(self, **kw):
|
| 23 |
+
table = super().version_table_impl(**kw)
|
| 24 |
+
table.c.version_num.type = String(128)
|
| 25 |
+
return table
|
| 26 |
+
|
| 27 |
+
|
| 28 |
def get_url() -> str:
|
| 29 |
return settings.database_url
|
| 30 |
|
|
|
|
| 52 |
poolclass=pool.NullPool,
|
| 53 |
)
|
| 54 |
|
| 55 |
+
with connectable.connect() as connection, connection.begin():
|
| 56 |
+
inspector = inspect(connection)
|
| 57 |
+
if inspector.has_table("alembic_version"):
|
| 58 |
+
column = next(c for c in inspector.get_columns("alembic_version") if c["name"] == "version_num")
|
| 59 |
+
length = getattr(column["type"], "length", None)
|
| 60 |
+
if length is not None and length < 128:
|
| 61 |
+
connection.execute(text("ALTER TABLE alembic_version ALTER COLUMN version_num TYPE varchar(128)"))
|
| 62 |
+
context.configure(connection=connection, target_metadata=target_metadata, compare_type=True)
|
| 63 |
with context.begin_transaction():
|
| 64 |
context.run_migrations()
|
| 65 |
|
api/auth-service/alembic/versions/0014_link_lab_art_routes.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Add source identities and fenced HTTP ingestion; preserve all legacy rows."""
|
| 2 |
+
from alembic import op
|
| 3 |
+
import sqlalchemy as sa
|
| 4 |
+
from sqlalchemy.dialects import postgresql as pg
|
| 5 |
+
|
| 6 |
+
revision = "0014_link_lab_art_routes"
|
| 7 |
+
down_revision = "0013_create_art_routing_tables"
|
| 8 |
+
branch_labels = None
|
| 9 |
+
depends_on = None
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def upgrade():
|
| 13 |
+
# This table previously existed only through the receiver's runtime DDL.
|
| 14 |
+
op.execute("""CREATE TABLE IF NOT EXISTS lab_route_library (
|
| 15 |
+
source text NOT NULL, source_id text NOT NULL, label text NOT NULL,
|
| 16 |
+
region_code text NOT NULL, region_name text NOT NULL, gpx_sha256 text NOT NULL,
|
| 17 |
+
accepted_at timestamptz NOT NULL, ai_protocol text NOT NULL,
|
| 18 |
+
collection_finalized boolean NOT NULL DEFAULT false,
|
| 19 |
+
gpx bytea NOT NULL, outline jsonb NOT NULL, active boolean NOT NULL DEFAULT true,
|
| 20 |
+
updated_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY(source, source_id))""")
|
| 21 |
+
for column in [sa.Column("source", sa.Text()), sa.Column("source_label", sa.Text()),
|
| 22 |
+
sa.Column("source_design_id", sa.Text()), sa.Column("source_design_version", sa.Text()),
|
| 23 |
+
sa.Column("design_digest", sa.String(64)), sa.Column("source_snapshot", pg.JSONB()),
|
| 24 |
+
sa.Column("design_ai_status", sa.Text(), nullable=False, server_default="unknown"),
|
| 25 |
+
sa.Column("design_ai_evidence", pg.JSONB()), sa.Column("svg_render_version", sa.Text()),
|
| 26 |
+
sa.Column("design_review_generation", sa.BigInteger())]:
|
| 27 |
+
op.add_column("art_templates", column)
|
| 28 |
+
op.create_unique_constraint("uq_art_templates_source_design", "art_templates",
|
| 29 |
+
["source", "source_label", "source_design_id", "design_digest"])
|
| 30 |
+
op.create_check_constraint("ck_art_templates_ai_status", "art_templates",
|
| 31 |
+
"design_ai_status IN ('unknown','ai_pass','ai_reject','ai_review')")
|
| 32 |
+
for column in [sa.Column("payload_sha256", sa.String(64)),
|
| 33 |
+
sa.Column("art_projection_version", sa.Integer(), nullable=False, server_default="0"),
|
| 34 |
+
sa.Column("sync_generation", sa.BigInteger(), nullable=False, server_default="0"),
|
| 35 |
+
sa.Column("revoked_generation", sa.BigInteger()), sa.Column("revoked_at", sa.DateTime(timezone=True))]:
|
| 36 |
+
op.add_column("lab_route_library", column)
|
| 37 |
+
for column in [sa.Column("source", sa.Text()), sa.Column("source_id", sa.Text()),
|
| 38 |
+
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
| 39 |
+
sa.Column("payload_sha256", sa.String(64)), sa.Column("gpx_sha256", sa.String(64)),
|
| 40 |
+
sa.Column("route_ai_status", sa.Text()), sa.Column("route_ai_evidence", pg.JSONB()),
|
| 41 |
+
sa.Column("road_verification", pg.JSONB()), sa.Column("graph_sha256", sa.String(64)),
|
| 42 |
+
sa.Column("region_name", sa.Text()), sa.Column("ai_protocol", sa.Text()),
|
| 43 |
+
sa.Column("accepted_at", sa.DateTime(timezone=True)), sa.Column("revoked_at", sa.DateTime(timezone=True)),
|
| 44 |
+
sa.Column("revocation_reason", sa.Text()), sa.Column("coordinate_order", sa.String(6)),
|
| 45 |
+
sa.Column("coordinate_crs", sa.Text()), sa.Column("sync_generation", sa.BigInteger()),
|
| 46 |
+
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now())]:
|
| 47 |
+
op.add_column("art_route_candidates", column)
|
| 48 |
+
op.create_unique_constraint("uq_art_routes_source_id", "art_route_candidates", ["source", "source_id"])
|
| 49 |
+
op.create_check_constraint("ck_art_routes_source_pair", "art_route_candidates", "(source IS NULL) = (source_id IS NULL)")
|
| 50 |
+
op.create_check_constraint("ck_art_routes_coordinate_order", "art_route_candidates", "coordinate_order IS NULL OR coordinate_order = 'latlon'")
|
| 51 |
+
op.create_foreign_key("fk_art_routes_library", "art_route_candidates", "lab_route_library",
|
| 52 |
+
["source", "source_id"], ["source", "source_id"], ondelete="RESTRICT")
|
| 53 |
+
op.create_table("lab_route_sync_state",
|
| 54 |
+
sa.Column("source", sa.Text(), primary_key=True),
|
| 55 |
+
sa.Column("generation", sa.BigInteger(), nullable=False, server_default="0"),
|
| 56 |
+
sa.Column("current_session_id", pg.UUID(as_uuid=True)),
|
| 57 |
+
sa.Column("v2_required", sa.Boolean(), nullable=False, server_default=sa.false()),
|
| 58 |
+
sa.Column("catalog_revision", sa.BigInteger(), nullable=False, server_default="0"),
|
| 59 |
+
sa.Column("last_full_generation", sa.BigInteger(), nullable=False, server_default="0"),
|
| 60 |
+
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()))
|
| 61 |
+
op.create_table("lab_route_sync_sessions",
|
| 62 |
+
sa.Column("id", pg.UUID(as_uuid=True), primary_key=True),
|
| 63 |
+
sa.Column("source", sa.Text(), sa.ForeignKey("lab_route_sync_state.source", ondelete="RESTRICT"), nullable=False),
|
| 64 |
+
sa.Column("client_request_id", pg.UUID(as_uuid=True), nullable=False),
|
| 65 |
+
sa.Column("generation", sa.BigInteger(), nullable=False),
|
| 66 |
+
sa.Column("mode", sa.Text(), nullable=False), sa.Column("status", sa.Text(), nullable=False),
|
| 67 |
+
sa.Column("manifest", pg.JSONB(), nullable=False), sa.Column("manifest_sha256", sa.String(64), nullable=False),
|
| 68 |
+
sa.Column("result", pg.JSONB()),
|
| 69 |
+
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
| 70 |
+
sa.UniqueConstraint("source", "client_request_id", name="uq_lab_sync_request"),
|
| 71 |
+
sa.UniqueConstraint("source", "generation", name="uq_lab_sync_generation"),
|
| 72 |
+
sa.CheckConstraint("mode IN ('full','partial')", name="ck_lab_sync_mode"),
|
| 73 |
+
sa.CheckConstraint("status IN ('open','completed','superseded')", name="ck_lab_sync_status"))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def downgrade():
|
| 77 |
+
op.drop_table("lab_route_sync_sessions")
|
| 78 |
+
op.drop_table("lab_route_sync_state")
|
| 79 |
+
op.drop_constraint("fk_art_routes_library", "art_route_candidates", type_="foreignkey")
|
| 80 |
+
for name in ["ck_art_routes_source_pair", "ck_art_routes_coordinate_order"]:
|
| 81 |
+
op.drop_constraint(name, "art_route_candidates", type_="check")
|
| 82 |
+
op.drop_constraint("uq_art_routes_source_id", "art_route_candidates", type_="unique")
|
| 83 |
+
for name in ["source", "source_id", "active", "payload_sha256", "gpx_sha256", "route_ai_status", "route_ai_evidence",
|
| 84 |
+
"road_verification", "graph_sha256", "region_name", "ai_protocol", "accepted_at", "revoked_at",
|
| 85 |
+
"revocation_reason", "coordinate_order", "coordinate_crs", "sync_generation", "updated_at"]:
|
| 86 |
+
op.drop_column("art_route_candidates", name)
|
| 87 |
+
for name in ["payload_sha256", "art_projection_version", "sync_generation", "revoked_generation", "revoked_at"]:
|
| 88 |
+
op.drop_column("lab_route_library", name)
|
| 89 |
+
op.drop_constraint("uq_art_templates_source_design", "art_templates", type_="unique")
|
| 90 |
+
op.drop_constraint("ck_art_templates_ai_status", "art_templates", type_="check")
|
| 91 |
+
for name in ["source", "source_label", "source_design_id", "source_design_version", "design_digest", "source_snapshot",
|
| 92 |
+
"design_ai_status", "design_ai_evidence", "svg_render_version", "design_review_generation"]:
|
| 93 |
+
op.drop_column("art_templates", name)
|
| 94 |
+
# Preserve the library, which may predate this migration and contain GPX evidence.
|
api/auth-service/requirements.txt
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
uvicorn[standard]
|
| 3 |
SQLAlchemy>=2.0
|
| 4 |
psycopg[binary]
|
| 5 |
-
alembic
|
| 6 |
pydantic-settings
|
| 7 |
python-jose[cryptography]
|
| 8 |
passlib[bcrypt]
|
|
|
|
| 2 |
uvicorn[standard]
|
| 3 |
SQLAlchemy>=2.0
|
| 4 |
psycopg[binary]
|
| 5 |
+
alembic>=1.14
|
| 6 |
pydantic-settings
|
| 7 |
python-jose[cryptography]
|
| 8 |
passlib[bcrypt]
|
deploy/huggingface/route-library-preflight.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run only inside the Space: verify database placement before applying 0014."""
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
|
| 8 |
+
import psycopg
|
| 9 |
+
|
| 10 |
+
REPORT = Path('/tmp/route-library-deployment-check.json')
|
| 11 |
+
REVISION = '0014_link_lab_art_routes'
|
| 12 |
+
TABLES = ('lab_route_library', 'art_templates', 'art_route_candidates')
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class DeploymentCheckError(RuntimeError):
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def connection_url(value):
|
| 20 |
+
return value.replace('postgresql+psycopg://', 'postgresql://', 1)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def inspect_database(url):
|
| 24 |
+
with psycopg.connect(connection_url(url), connect_timeout=10,
|
| 25 |
+
options='-c default_transaction_read_only=on -c statement_timeout=15000') as conn:
|
| 26 |
+
identity = conn.execute('SELECT current_database(),current_schema(),inet_server_addr()::text,inet_server_port()').fetchone()
|
| 27 |
+
relations = {name: conn.execute('SELECT to_regclass(%s)::oid', (name,)).fetchone()[0] for name in TABLES}
|
| 28 |
+
counts = {name: conn.execute('SELECT count(*) FROM ' + name).fetchone()[0] if relations[name] else None for name in TABLES}
|
| 29 |
+
versions = ([r[0] for r in conn.execute('SELECT version_num FROM alembic_version')]
|
| 30 |
+
if conn.execute("SELECT to_regclass('alembic_version')").fetchone()[0] else [])
|
| 31 |
+
return identity, relations, counts, versions
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def verify_placement():
|
| 35 |
+
auth = os.environ.get('DATABASE_URL') or os.environ.get('AUTH_DATABASE_URL')
|
| 36 |
+
library = os.environ.get('ROUTE_LIBRARY_DATABASE_URL') or auth
|
| 37 |
+
catalog = os.environ.get('ART_ROUTE_CATALOG_DATABASE_URL') or library
|
| 38 |
+
if not auth or not library or not catalog:
|
| 39 |
+
raise DeploymentCheckError('database_configuration_missing')
|
| 40 |
+
if os.environ.get('ART_ROUTE_CATALOG_JSON'):
|
| 41 |
+
raise DeploymentCheckError('file_catalog_overrides_database')
|
| 42 |
+
observations = [inspect_database(url) for url in (auth, library, catalog)]
|
| 43 |
+
if any(value[:2] != observations[0][:2] for value in observations[1:]):
|
| 44 |
+
raise DeploymentCheckError('database_or_schema_mismatch')
|
| 45 |
+
return observations[0][2:]
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def run_alembic():
|
| 49 |
+
env = dict(os.environ)
|
| 50 |
+
env['PGOPTIONS'] = '-c lock_timeout=5000 -c statement_timeout=120000'
|
| 51 |
+
result = subprocess.run([sys.executable, '-m', 'alembic', 'upgrade', REVISION],
|
| 52 |
+
cwd='/app/api/auth-service', env=env, capture_output=True, timeout=180)
|
| 53 |
+
# Migration exceptions can contain credentials and SQL values. Never forward them.
|
| 54 |
+
if result.returncode:
|
| 55 |
+
raise DeploymentCheckError('migration_failed_details_suppressed')
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def check_and_migrate(migrate=run_alembic):
|
| 59 |
+
before, versions_before = verify_placement()
|
| 60 |
+
migrate()
|
| 61 |
+
after, versions_after = verify_placement()
|
| 62 |
+
if versions_after != [REVISION] or any(after[name] is None for name in TABLES):
|
| 63 |
+
raise DeploymentCheckError('migration_or_tables_not_ready')
|
| 64 |
+
if any(before[name] is not None and before[name] != after[name] for name in TABLES):
|
| 65 |
+
raise DeploymentCheckError('existing_row_counts_changed')
|
| 66 |
+
return {'ready': True, 'same_database': True, 'catalog_uses_database': True,
|
| 67 |
+
'migration_before': versions_before, 'migration_after': versions_after,
|
| 68 |
+
'tables': {name: True for name in TABLES}, 'row_counts_before': before,
|
| 69 |
+
'row_counts_after': after, 'existing_rows_preserved': True}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
if __name__ == '__main__':
|
| 73 |
+
try:
|
| 74 |
+
result = check_and_migrate()
|
| 75 |
+
descriptor = os.open(REPORT, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600)
|
| 76 |
+
with os.fdopen(descriptor, 'w') as output:
|
| 77 |
+
json.dump(result, output)
|
| 78 |
+
print(json.dumps({'deployment_preflight': 'passed', **result}), flush=True)
|
| 79 |
+
except Exception as exc:
|
| 80 |
+
code = str(exc) if isinstance(exc, DeploymentCheckError) else type(exc).__name__
|
| 81 |
+
print(json.dumps({'deployment_preflight': 'failed', 'error': code}), flush=True)
|
| 82 |
+
raise SystemExit(1) from None
|
deploy/huggingface/start-hf.sh
CHANGED
|
@@ -134,11 +134,8 @@ Path("/app/main/public/runtime-env.js").write_text(
|
|
| 134 |
)
|
| 135 |
PY
|
| 136 |
|
| 137 |
-
echo "
|
| 138 |
-
|
| 139 |
-
cd /app/api/auth-service
|
| 140 |
-
"${PYTHON_BIN}" -m alembic upgrade head
|
| 141 |
-
)
|
| 142 |
|
| 143 |
start_bg "auth-service" bash -c "cd /app/api/auth-service && exec \"${PYTHON_BIN}\" -m uvicorn app.main:app --host 127.0.0.1 --port 8001"
|
| 144 |
start_bg "ai-service" bash -c "cd /app/api/ai-service && exec \"${PYTHON_BIN}\" -m uvicorn app.main:app --host 127.0.0.1 --port 8002"
|
|
|
|
| 134 |
)
|
| 135 |
PY
|
| 136 |
|
| 137 |
+
echo "Checking server database placement and applying route library migration..."
|
| 138 |
+
"${PYTHON_BIN}" /app/route-library-preflight.py
|
|
|
|
|
|
|
|
|
|
| 139 |
|
| 140 |
start_bg "auth-service" bash -c "cd /app/api/auth-service && exec \"${PYTHON_BIN}\" -m uvicorn app.main:app --host 127.0.0.1 --port 8001"
|
| 141 |
start_bg "ai-service" bash -c "cd /app/api/ai-service && exec \"${PYTHON_BIN}\" -m uvicorn app.main:app --host 127.0.0.1 --port 8002"
|
docs/LAB_GPX_SYNC.md
CHANGED
|
@@ -1,5 +1,140 @@
|
|
| 1 |
# Lab 승인 GPX → DemoService 반영
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
Lab 전국 수집기의 `accepted JOIN items`에 있는 승인 경로만 동기화한다. Lab의
|
| 4 |
`inventory.sqlite3`와 router 일반 Worker의 `routes.sqlite3`는 서로 다른
|
| 5 |
자료다. `tools/sync_lab_gpx.py`는 전자를 읽고, DemoService의 AI 서비스가
|
|
|
|
| 1 |
# Lab 승인 GPX → DemoService 반영
|
| 2 |
|
| 3 |
+
## 운영 배포와 부분 시험 확인
|
| 4 |
+
|
| 5 |
+
실제 Lab 설치본의 보호 기능을 포함한 송신기는 `tools/lab_sync/sync_lab_gpx.py`에
|
| 6 |
+
보존한다. 기존 상태 파일·잠금 파일을 그대로 사용하며 v2는 명시적으로 선택한다.
|
| 7 |
+
|
| 8 |
+
Hugging Face 시작 스크립트는 서비스 시작 전에 서버 안에서 migration·수신·카탈로그의
|
| 9 |
+
실제 PostgreSQL DB와 스키마·테이블 식별자가 일치하는지 검사한다. 다르면 migration을
|
| 10 |
+
실행하지 않는다. 동일함을 확인한 뒤 migration 0014를 적용하고 기존 행 수 보존을
|
| 11 |
+
검사한다. 접속 문자열과 원본 자료는 출력하지 않는다.
|
| 12 |
+
|
| 13 |
+
같은 수신 인증을 요구하는 `GET /deployment-status`는 배포 GitHub SHA, 실제 migration
|
| 14 |
+
revision, 세 표의 존재 및 서버의 동일 DB 사전 검사 결과만 반환한다.
|
| 15 |
+
`GET /routes/{source_id}/storage`는 저장된 세 표를 서버의 읽기 전용 트랜잭션으로 조회하여
|
| 16 |
+
실제 두 art ID·내용 해시·활성 상태와 GPX/저장 좌표 일치 여부를 반환한다.
|
| 17 |
+
원본 GPX·좌표·도안 본문이나 DB 접속정보는 응답에 포함하지 않는다.
|
| 18 |
+
|
| 19 |
+
실제 한 건 시험은 현재 accepted 자료의 `--probe-id`로 부분 세션을 사용한다.
|
| 20 |
+
저장 응답을 확인한 뒤 storage 조회의 실제 ID와 비교한다. 전체 승인 스냅샷을
|
| 21 |
+
보내지 않고 반복 실행이나 대량 전송을 자동으로 시작하지 않는다.
|
| 22 |
+
|
| 23 |
+
## v2: 도안·라이브러리·경로 후보를 함께 저장
|
| 24 |
+
|
| 25 |
+
`--contract-version 2`는 Lab에서 HTTP API만 사용한다. Lab의 읽기 전용
|
| 26 |
+
SQLite 승인 조회와 별도의 로컬 전송 상태 SQLite 외에는 DB에 접속하지 않는다.
|
| 27 |
+
DemoService PostgreSQL 드라이버·접속 문자열은 Lab에 필요하지 않다.
|
| 28 |
+
기존 CLI와 요청은 기본적으로 v1을 유지하며, v2 전환은 명시적으로 수행한다.
|
| 29 |
+
|
| 30 |
+
서버는 기존 인증·서버 DB 설정으로 다음을 **경로 1건당 한 트랜잭션**에서 저장한다.
|
| 31 |
+
|
| 32 |
+
1. 원본 DesignSpec digest를 검증하고 `art_templates`를 확인·등록한다.
|
| 33 |
+
2. `lab_route_library`에 GPX와 위치 없는 XY 외곽선을 저장한다.
|
| 34 |
+
3. GPX에서 직접 파싱한 WGS84 `[위도, 경도]`만 `art_route_candidates.coordinates`에 저장한다.
|
| 35 |
+
|
| 36 |
+
도안 SVG는 원본 도안 점들로 생성한다. HTTP 요청에 있는 경로·파일명을 서버에서
|
| 37 |
+
열지 않는다. v2는 클라이언트가 직접 지정하는 후보 GPS 배열과 SVG 본문을 받지 않는다.
|
| 38 |
+
도안 자체의 `design_ai_status`와 경로의 AI·도로 검증을 별도로 저장한다.
|
| 39 |
+
`ai_reject` 도안도 승인 경로의 원본 참조로 보존할 수 있지만 승인 도안으로 승격하지 않는다.
|
| 40 |
+
`ai_pass` 도안을 주장하려면 해당 도안 자체의 두 모델 검수 근거도 필요하다.
|
| 41 |
+
|
| 42 |
+
### v2 요청 필드
|
| 43 |
+
|
| 44 |
+
기존 GPX·라벨·지역·승인 시각·검수 정책 필드에 다음을 추가한다.
|
| 45 |
+
|
| 46 |
+
| 필드 | 의미 |
|
| 47 |
+
|---|---|
|
| 48 |
+
| `contract_version` | `2` |
|
| 49 |
+
| `session_id`, `generation` | 서버가 발급한 현재 출처 세션·세대 |
|
| 50 |
+
| `payload_sha256` | 아래 규칙으로 계산한 의미상 내용 해시 |
|
| 51 |
+
| `template.design_id`, `source_version`, `design_digest` | 원본 도안 식별 |
|
| 52 |
+
| `template.snapshot` | digest 검증에 사용할 원본 DesignSpec 전체 JSON |
|
| 53 |
+
| `template.design_ai_status`, `design_ai_evidence` | 원본 도안의 독립 AI 판정 |
|
| 54 |
+
| `route_ai_evidence` | 경로의 실제 두 모델 검수 근거 |
|
| 55 |
+
| `road_verification`, `graph_sha256` | 저장된 도로 검사와 그래프 버전 근거 |
|
| 56 |
+
| `distance_m`, `validator_version` | 원본 도로 길이의 정수 미터와 검증 정책 버전 |
|
| 57 |
+
| `previous_payload_sha256` | 명시적 갱신의 비교·교환 조건. 기본값 없음 |
|
| 58 |
+
| `reapprove` | 취소 후 명시적 재승인. 기본값 false |
|
| 59 |
+
|
| 60 |
+
내용 해시는 키 정렬, UTF-8, `ensure_ascii=False`, `allow_nan=False`,
|
| 61 |
+
구분자 `,`·`:`인 JSON의 SHA-256이다. `session_id`, `generation`,
|
| 62 |
+
`payload_sha256`, `gpx_base64`, `previous_payload_sha256`, `reapprove`는 제외한다.
|
| 63 |
+
GPX 바이트는 본문 대신 검증된 `sha256` 필드로 해시에 연결된다.
|
| 64 |
+
나머지 기본 필드는 생략하지 않고 송신기와 서버 모델의 값이 일치해야 한다.
|
| 65 |
+
|
| 66 |
+
### 세션과 승인 취소
|
| 67 |
+
|
| 68 |
+
- `GET /sync-state`: 계약 버전·현재 세대·카탈로그 개정 번호만 반환한다.
|
| 69 |
+
- `POST /sync-sessions`: `client_request_id`, `expected_generation`, `mode`,
|
| 70 |
+
원본 경로 ID → 내용 해시인 `manifest`, `manifest_sha256`으로 시작한다.
|
| 71 |
+
- `PUT /routes/{source_id}`: 세션 manifest와 내용 해시가 일치하는 자료만 저장한다.
|
| 72 |
+
- `GET /sync-sessions/{session_id}`: 현재 세션의 저장 상태를 조회한다.
|
| 73 |
+
- `POST /sync-sessions/{session_id}/complete?generation=...`: 부분 전송을 완료한다.
|
| 74 |
+
다른 자료를 비활성화하지 않는다.
|
| 75 |
+
- 기존 `POST /approval-snapshot`: v2에서는 `session_id`, `generation`,
|
| 76 |
+
`manifest_sha256`도 받���다. 전체 세션의 모든 art 연결이 확인된 경우에만
|
| 77 |
+
라이브러리와 후보 경로를 함께 비활성화한다.
|
| 78 |
+
|
| 79 |
+
새 세션은 이전 세션을 차단한다. 세션 시작 요청은 로컬 상태에 먼저 저장한
|
| 80 |
+
`client_request_id`로 재시도하며, 응답 유실 때문에 세대를 새로 발급하지 않는다.
|
| 81 |
+
이미 완료된 동일 스냅샷의 재시도도 같은 결과를 반환한다.
|
| 82 |
+
기존의 대량 비활성화 방지 제한을 유지하며, 부분 목록을 전체 목록으로 보내면 안 된다.
|
| 83 |
+
|
| 84 |
+
### 멱등 저장과 명시적 변경
|
| 85 |
+
|
| 86 |
+
같은 ID·같은 내용은 `result=unchanged`로 실제 저장 UUID를 돌려주고,
|
| 87 |
+
업무 행의 `updated_at`도 갱신하지 않는다. 다른 내용은 기본적으로 409다.
|
| 88 |
+
`--allow-updates`는 로컬에 보존된 직전 저장 응답의 해시로 비교·교환을 요청한다.
|
| 89 |
+
로컬 이력이 없으면 이 옵션만으로 임의의 서버 내용을 덮어쓸 수 없다.
|
| 90 |
+
|
| 91 |
+
취소된 경로는 `--allow-reapproval`과 이전보다 늦은 원본 승인 시각이 필요하다.
|
| 92 |
+
v2로 저장된 내용도 달라졌다면 직전 내용 해시 조건을 함께 만족해야 한다.
|
| 93 |
+
오래된 세션은 이 옵션들과 관계없이 거절한다.
|
| 94 |
+
|
| 95 |
+
성공 응답은 커밋 후의 `art_template_id`, `art_route_candidate_id`,
|
| 96 |
+
`design_ai_status`, `route_validation_status`, `payload_sha256`, `contract_version`을
|
| 97 |
+
포함한다. 송신기는 이 응답을 확인한 뒤에만 `http_v2_sent`에 완료 기록을 남긴다.
|
| 98 |
+
옛 `sent` 테이블의 기록을 v2 저장 성공으로 간주하지 않는다.
|
| 99 |
+
중단된 같은 세션은 확인된 요청부터 이어서 처리한다. 새 세션에서는 기존 자료도
|
| 100 |
+
HTTP 멱등 저장으로 다시 확인하므로 반복 실행 간격을 운영 규모에 맞춰 정해야 한다.
|
| 101 |
+
|
| 102 |
+
### 호환성과 사전 조건
|
| 103 |
+
|
| 104 |
+
기존 v1 요청·응답 형식을 유지한다. 다만 한 출처가 v2 관리로 전환된 뒤에는
|
| 105 |
+
v1의 동일 활성 자료 확인만 허용하고, 새 쓰기·갱신·스냅샷은 409로 차단한다.
|
| 106 |
+
v1 요청에는 세대·원본 도안 정보가 없으므로 강한 순서 보장이나 art 저장 성공을
|
| 107 |
+
대신 주장하지 않는다. v1에서도 취소된 경로를 재전송만으로 재활성화하지 않는다.
|
| 108 |
+
|
| 109 |
+
운영에서는 세 업무 테이블이 같은 서버 DB에 존재하고 migration 0014가 적용되어야 한다.
|
| 110 |
+
요청 중에는 테이블을 생성하지 않는다. 같은 도안 분류의 내부 정수 버전은 서버가
|
| 111 |
+
잠금 아래 배정하고, 원본 ID·버전·digest와 별도로 보존한다.
|
| 112 |
+
이전 Alembic 이력은 `routes`·`run_sessions` 기초 테이블의 존재를 전제로 한다.
|
| 113 |
+
버전 문자열이 32자를 넘으므로 버전 테이블은 128자를 사용하며 기존 짧은 컬럼은
|
| 114 |
+
마이그레이션 실행 시 데이터 보존 방식으로 넓힌다.
|
| 115 |
+
|
| 116 |
+
아래는 향후 검증 환경에서 사용할 형태이며, 실제 승인 자료로 실행한 명령이 아니다.
|
| 117 |
+
|
| 118 |
+
```bash
|
| 119 |
+
python tools/sync_lab_gpx.py \
|
| 120 |
+
--root /path/to/synthetic-lab \
|
| 121 |
+
--url https://example.invalid/api/v1/ai/internal/route-library \
|
| 122 |
+
--token-file /path/to/existing-auth-file \
|
| 123 |
+
--contract-version 2 --check-only
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
`--check-only`는 인증 파일을 읽거나 HTTP 요청을 보내지 않는다.
|
| 127 |
+
실제 시험 전송은 별도 허용 후 `--probe-id` 또는 `--max-routes`로 범위를 제한한다.
|
| 128 |
+
이 두 옵션은 부분 세션을 사용하여 승인 취소를 보내지 않는다.
|
| 129 |
+
`--new-session`은 오래된 미완료 세션 대신 현재 승인 자료로 새 세션을 시작하는
|
| 130 |
+
명시적 복구 옵션이다. 동시 송신기는 상태 파일별 잠금으로 차단한다.
|
| 131 |
+
|
| 132 |
+
이 저장소의 변경은 기존 Lab 실행 파일을 자동으로 교체하지 않는다.
|
| 133 |
+
강화 송신기를 교체할 때는 기존 상태 파일·승인 가드·운영 제한을 보존하고,
|
| 134 |
+
기존 프로세스와의 동시 실행을 막는 전환 절차를 별도로 확인해야 한다.
|
| 135 |
+
|
| 136 |
+
## 기존 v1 연동
|
| 137 |
+
|
| 138 |
Lab 전국 수집기의 `accepted JOIN items`에 있는 승인 경로만 동기화한다. Lab의
|
| 139 |
`inventory.sqlite3`와 router 일반 Worker의 `routes.sqlite3`는 서로 다른
|
| 140 |
자료다. `tools/sync_lab_gpx.py`는 전자를 읽고, DemoService의 AI 서비스가
|
docs/LOCAL_ART_ROUTE_CATALOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
| 1 |
# 로컬 검증 경로 카탈로그와 모양 자동 추천
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
경로 생성 화면의 기본값인 **모양 상관없음 · 자동 추천**은 내장 예시와 검증 경로
|
| 4 |
카탈로그를 같은 주변 보행망에서 비교한다. 도로 투영으로 먼저 후보 도안을 추리고,
|
| 5 |
거리별 우선 두 모양을 실제 도로망에 연결하고, 통과한 경로가 부족하면 최대 네 모양까지
|
|
|
|
| 1 |
# 로컬 검증 경로 카탈로그와 모양 자동 추천
|
| 2 |
|
| 3 |
+
## HTTP v2 연동 자료의 조회
|
| 4 |
+
|
| 5 |
+
v2 후보는 `art_route_candidates.active=true`, 경로 검증 `passed`, 연결된
|
| 6 |
+
`lab_route_library.active=true`, 양쪽의 동일 내용 해시를 확인한 뒤 읽는다.
|
| 7 |
+
도안 자체의 AI 판정은 경로 승인과 별개이며, 경로 통과를 도안 승인 표시로 쓰지 않는다.
|
| 8 |
+
기존 출처 미상 후보는 원래 조회 정책을 유지하고 도안 판정은 `unknown`으로 보존한다.
|
| 9 |
+
|
| 10 |
+
v2 행은 `[위도, 경도]`·WGS84를 명시한다. 기존 행의 순서는 기존 환경변수로
|
| 11 |
+
해석한다. 위치 없는 라이브러리 외곽선은 이미 XY이므로 GPS 변환기에 넣지 않는다.
|
| 12 |
+
라이브러리와 카탈로그에 같은 외곽선이 있을 때는 동일 좌표열을 중복 제거한다.
|
| 13 |
+
|
| 14 |
+
카탈로그 DB 결과는 요청마다 읽어 다른 프로세스의 취소를 반영한다. 라이브러리
|
| 15 |
+
캐시는 공유 `catalog_revision`을 확인하며 DB 장애 시 이전 외부 캐시를 재사용하지 않는다.
|
| 16 |
+
자동 추천도 후보 선별 전에 라이브러리 기여분을 갱신한다.
|
| 17 |
+
생성·점 탐색 처리 중 카탈로그 개정 번호가 바뀌면 이전 자료로 만든 결과를
|
| 18 |
+
반환하지 않고 `409 route_library_changed_retry`를 반환한다.
|
| 19 |
+
|
| 20 |
+
`ART_ROUTE_CATALOG_DATABASE_URL`이 없으면 기존 `ROUTE_LIBRARY_DATABASE_URL`을
|
| 21 |
+
사용한다. Lab에는 이 연결 정보를 전달하지 않는다. `ART_ROUTE_CATALOG_JSON`은
|
| 22 |
+
계속 파일 우선 설정이며, 파일 스냅샷의 취소 반영은 별도 재생성이 필요하다.
|
| 23 |
+
기존 200행 제한과 화면별 라벨 목록은 이 변경에서 확대하지 않는다.
|
| 24 |
+
|
| 25 |
+
## 기존 조회 설정
|
| 26 |
+
|
| 27 |
경로 생성 화면의 기본값인 **모양 상관없음 · 자동 추천**은 내장 예시와 검증 경로
|
| 28 |
카탈로그를 같은 주변 보행망에서 비교한다. 도로 투영으로 먼저 후보 도안을 추리고,
|
| 29 |
거리별 우선 두 모양을 실제 도로망에 연결하고, 통과한 경로가 부족하면 최대 네 모양까지
|
tools/lab_sync/README.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Guarded Lab HTTP sender
|
| 2 |
+
|
| 3 |
+
This is the prepared Lab installation sender. It preserves the original
|
| 4 |
+
`~/.local/share/demoservice-sync/state.sqlite3`, `transfer.lock`, approval guards,
|
| 5 |
+
one worker, cumulative route cap, and halt-on-error policy. Its default contract
|
| 6 |
+
remains v1; use `--contract-version 2` explicitly for the server-managed session
|
| 7 |
+
contract. Never run an old sender concurrently with this installation.
|
| 8 |
+
|
| 9 |
+
The sender reads only the local Lab approval inventory and its local transfer
|
| 10 |
+
ledger. DemoService PostgreSQL connections, schema migration, and storage
|
| 11 |
+
verification run on the server. No server DB credentials belong on the Lab.
|
| 12 |
+
|
| 13 |
+
`--probe-id` sends one currently accepted route in a partial session. It does not
|
| 14 |
+
send an approval snapshot. Existing v1 ledger rows never count as v2 art storage;
|
| 15 |
+
both actual art UUIDs and the content hash must be verified before a v2 receipt
|
| 16 |
+
is recorded. A read-only `--check-only` reads no credential and makes no HTTP
|
| 17 |
+
request or local state/lock mutation.
|
| 18 |
+
|
| 19 |
+
Synthetic offline tests live in `tests/`; the copied approval policy and guard
|
| 20 |
+
SQL there are test fixtures. Tests block network connections and use only
|
| 21 |
+
temporary synthetic inventories, records, designs, GPX files, and state files.
|
tools/lab_sync/sync_lab_gpx.py
ADDED
|
@@ -0,0 +1,767 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Push one consistent Lab approval snapshot to the DemoService route library.
|
| 2 |
+
|
| 3 |
+
Run on the Lab host: python tools/sync_lab_gpx.py --root /home/jovyan/artifacts/...
|
| 4 |
+
The receiver must already be deployed with ROUTE_LIBRARY_TOKEN configured.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import base64
|
| 10 |
+
from contextlib import contextmanager
|
| 11 |
+
from datetime import datetime, timezone
|
| 12 |
+
import fcntl
|
| 13 |
+
from functools import lru_cache
|
| 14 |
+
import hashlib
|
| 15 |
+
import importlib.util
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
import os
|
| 19 |
+
import re
|
| 20 |
+
import signal
|
| 21 |
+
import sqlite3
|
| 22 |
+
import time
|
| 23 |
+
import urllib.error
|
| 24 |
+
import urllib.request
|
| 25 |
+
import urllib.parse
|
| 26 |
+
import uuid
|
| 27 |
+
import xml.etree.ElementTree as ET
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
DEFAULT_STATE = Path.home() / ".local/share/demoservice-sync/state.sqlite3"
|
| 31 |
+
LOCK_PATH = DEFAULT_STATE.parent / "transfer.lock"
|
| 32 |
+
STATUS_PATH = DEFAULT_STATE.parent / "status.json"
|
| 33 |
+
HALT_PATH = DEFAULT_STATE.parent / "HALTED.json"
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class ReceiverUnhealthy(RuntimeError):
|
| 37 |
+
pass
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class ApprovalChanged(RuntimeError):
|
| 41 |
+
pass
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def write_status(path: Path, value: dict) -> None:
|
| 45 |
+
data = {"updated_at": datetime.now(timezone.utc).isoformat(), "pid": os.getpid(), **value}
|
| 46 |
+
temporary = path.with_name('.'+path.name+'.'+str(os.getpid()))
|
| 47 |
+
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600)
|
| 48 |
+
try:
|
| 49 |
+
with os.fdopen(descriptor, 'w') as output:
|
| 50 |
+
json.dump(data, output, ensure_ascii=False)
|
| 51 |
+
output.write('\n')
|
| 52 |
+
os.replace(temporary, path)
|
| 53 |
+
finally:
|
| 54 |
+
temporary.unlink(missing_ok=True)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def check_health(destination: str) -> None:
|
| 58 |
+
parts = urllib.parse.urlsplit(destination)
|
| 59 |
+
url = urllib.parse.urlunsplit((parts.scheme, parts.netloc, '/health', '', ''))
|
| 60 |
+
with urllib.request.urlopen(url, timeout=30) as response:
|
| 61 |
+
value = json.load(response)
|
| 62 |
+
if not isinstance(value, dict) or value.get('status') != 'ok':
|
| 63 |
+
raise ReceiverUnhealthy('DemoService health is abnormal')
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class AlreadyRunning(RuntimeError):
|
| 67 |
+
pass
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@contextmanager
|
| 71 |
+
def single_instance(lock_path: Path = LOCK_PATH):
|
| 72 |
+
"""Keep the same inode locked across every cycle, sleep, and retry."""
|
| 73 |
+
lock_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
| 74 |
+
descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600)
|
| 75 |
+
try:
|
| 76 |
+
try:
|
| 77 |
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 78 |
+
except BlockingIOError:
|
| 79 |
+
raise AlreadyRunning("Another GPX transfer is already running") from None
|
| 80 |
+
os.fchmod(descriptor, 0o600)
|
| 81 |
+
os.ftruncate(descriptor, 0)
|
| 82 |
+
os.write(descriptor, json.dumps({"pid": os.getpid(), "started_at": datetime.now(timezone.utc).isoformat()}).encode())
|
| 83 |
+
yield
|
| 84 |
+
finally:
|
| 85 |
+
os.close(descriptor)
|
| 86 |
+
# Never unlink or replace the lock file: that permits two locked inodes.
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
@lru_cache(maxsize=8)
|
| 90 |
+
def approval_validator(root: Path):
|
| 91 |
+
policy = root / "approval_policy.py"
|
| 92 |
+
spec = importlib.util.spec_from_file_location("lab_approval_policy", policy)
|
| 93 |
+
if spec is None or spec.loader is None:
|
| 94 |
+
raise RuntimeError("Approval policy is unavailable")
|
| 95 |
+
module = importlib.util.module_from_spec(spec)
|
| 96 |
+
spec.loader.exec_module(module)
|
| 97 |
+
return module.valid_review
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def approved_rows(root: Path, source_id: str | None = None, *, art: bool = False) -> list[sqlite3.Row]:
|
| 101 |
+
database = (root / "inventory.sqlite3").resolve()
|
| 102 |
+
connection = sqlite3.connect(f"file:{database.as_posix()}?mode=ro", uri=True)
|
| 103 |
+
connection.row_factory = sqlite3.Row
|
| 104 |
+
try:
|
| 105 |
+
connection.execute("PRAGMA query_only=ON")
|
| 106 |
+
connection.execute("BEGIN")
|
| 107 |
+
version = connection.execute("SELECT value FROM settings WHERE key='approval_guards_version'").fetchone()
|
| 108 |
+
if version is None or version[0] != "1":
|
| 109 |
+
raise RuntimeError("Approval guards must be installed before synchronization")
|
| 110 |
+
scope = " WHERE a.item_id=?" if source_id is not None else ""
|
| 111 |
+
parameters = (source_id,) if source_id is not None else ()
|
| 112 |
+
invalid = connection.execute("""
|
| 113 |
+
SELECT count(*) FROM accepted AS a
|
| 114 |
+
LEFT JOIN eligible_approval_items AS i ON i.id=a.item_id
|
| 115 |
+
AND i.label IS a.label AND i.city_id IS a.city_id AND i.design_id IS a.design_id
|
| 116 |
+
LEFT JOIN approval_revocations AS r ON r.item_id=a.item_id
|
| 117 |
+
""" + scope + (" AND " if scope else " WHERE ") + "(i.id IS NULL OR r.item_id IS NOT NULL)", parameters).fetchone()[0]
|
| 118 |
+
if invalid:
|
| 119 |
+
raise RuntimeError("Invalid or revoked entries remain in accepted; refusing the entire snapshot")
|
| 120 |
+
extra = ", i.kind, i.design_id, i.record_path" if art else ""
|
| 121 |
+
rows = connection.execute("""
|
| 122 |
+
SELECT i.id, i.label, i.city_id, i.city_name, i.ai_json,
|
| 123 |
+
i.geometry_ok, i.seoul_ok, i.status, a.accepted_at
|
| 124 |
+
""" + extra + """
|
| 125 |
+
FROM accepted AS a JOIN items AS i ON i.id = a.item_id
|
| 126 |
+
""" + scope + " ORDER BY i.id", parameters).fetchall()
|
| 127 |
+
validate = approval_validator(root)
|
| 128 |
+
if any(not validate(row['ai_json'], row['label']) for row in rows):
|
| 129 |
+
raise RuntimeError("Two-model approval evidence is invalid; refusing the entire snapshot")
|
| 130 |
+
connection.commit()
|
| 131 |
+
return rows
|
| 132 |
+
finally:
|
| 133 |
+
connection.close()
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def prepare_route(root: Path, row: sqlite3.Row, finalized: bool) -> tuple[bytes, dict, str]:
|
| 137 |
+
source_id, label = row['id'], row['label']
|
| 138 |
+
if (not isinstance(source_id, str) or len(source_id) != 64
|
| 139 |
+
or any(c not in '0123456789abcdef' for c in source_id)
|
| 140 |
+
or label not in {'dog','heart','tree','tshirt','candle','pants','star'}):
|
| 141 |
+
raise RuntimeError("Unexpected route ID or label in approval list")
|
| 142 |
+
if row['status'] != 'ai_pass' or row['geometry_ok'] != 1 or row['seoul_ok'] != 1:
|
| 143 |
+
raise RuntimeError("Approval state is invalid")
|
| 144 |
+
gpx = (root/'gpx'/label/(source_id+'.gpx')).read_bytes()
|
| 145 |
+
metadata = {
|
| 146 |
+
'label': label, 'region_code': row['city_id'], 'region_name': row['city_name'],
|
| 147 |
+
'sha256': hashlib.sha256(gpx).hexdigest(), 'accepted_at': row['accepted_at'],
|
| 148 |
+
'review_status': row['status'], 'ai_protocol': json.loads(row['ai_json'])['protocol'],
|
| 149 |
+
'geometry_ok': True, 'seoul_ok': True, 'collection_finalized': finalized,
|
| 150 |
+
}
|
| 151 |
+
signature = hashlib.sha256(json.dumps(metadata, sort_keys=True).encode()).hexdigest()
|
| 152 |
+
return gpx, metadata, signature
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def check_only(root: Path, state_path: Path) -> dict:
|
| 156 |
+
"""Read-only preflight: no token read, no HTTP requests, no DB writes."""
|
| 157 |
+
rows = approved_rows(root)
|
| 158 |
+
if not rows:
|
| 159 |
+
raise RuntimeError("Empty approval list; refusing to synchronize")
|
| 160 |
+
finalized = (root/'completion.json').is_file()
|
| 161 |
+
signatures = {row['id']: prepare_route(root, row, finalized)[2] for row in rows}
|
| 162 |
+
with sqlite3.connect(state_path.resolve().as_uri()+'?mode=ro', uri=True) as state:
|
| 163 |
+
sent = dict(state.execute('SELECT id,signature FROM sent'))
|
| 164 |
+
return {'approved': len(rows), 'sent': len(sent),
|
| 165 |
+
'unchanged_sent': sum(signatures.get(k) == v for k,v in sent.items()),
|
| 166 |
+
'changed_sent': sum(k in signatures and signatures[k] != v for k,v in sent.items()),
|
| 167 |
+
'revoked_sent': len(set(sent)-set(signatures)),
|
| 168 |
+
'unsent_approved': len(set(signatures)-set(sent)), 'http_requests': 0}
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def send(url: str, token: str, method: str, payload: dict | None) -> dict:
|
| 172 |
+
request = urllib.request.Request(
|
| 173 |
+
url, json.dumps(payload, allow_nan=False, separators=(",", ":")).encode() if payload is not None else None,
|
| 174 |
+
{"Authorization": "Bearer " + token, "Content-Type": "application/json"}, method=method,
|
| 175 |
+
)
|
| 176 |
+
with urllib.request.urlopen(request, timeout=30) as response:
|
| 177 |
+
return json.load(response)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def canonical(value):
|
| 181 |
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def content_sha(payload):
|
| 185 |
+
data = dict(payload)
|
| 186 |
+
for key in ("session_id", "generation", "payload_sha256", "gpx_base64", "previous_payload_sha256", "reapprove"):
|
| 187 |
+
data.pop(key, None)
|
| 188 |
+
return hashlib.sha256(canonical(data).encode()).hexdigest()
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def approved_art_rows(root, source_id=None):
|
| 192 |
+
"""Reuse the installation's approval guards and semantic two-model policy."""
|
| 193 |
+
return [dict(row) for row in approved_rows(root, source_id, art=True)]
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def verify_source_gpx(raw, expected):
|
| 197 |
+
try:
|
| 198 |
+
if not 0 < len(raw) <= 128 * 1024 or b"<!DOCTYPE" in raw.upper() or b"<!ENTITY" in raw.upper():
|
| 199 |
+
raise ValueError
|
| 200 |
+
document = ET.fromstring(raw)
|
| 201 |
+
tracks = [n for n in document.iter() if n.tag.rsplit("}", 1)[-1] == "trk"]
|
| 202 |
+
segments = [n for n in document.iter() if n.tag.rsplit("}", 1)[-1] == "trkseg"]
|
| 203 |
+
if document.tag.rsplit("}", 1)[-1] != "gpx" or len(tracks) != 1 or len(segments) != 1 or segments[0] not in list(tracks[0]):
|
| 204 |
+
raise ValueError
|
| 205 |
+
points = [(float(n.attrib["lat"]), float(n.attrib["lon"])) for n in segments[0]
|
| 206 |
+
if n.tag.rsplit("}", 1)[-1] == "trkpt"]
|
| 207 |
+
if not 12 <= len(points) <= 10000 or len(points) != len(expected):
|
| 208 |
+
raise ValueError
|
| 209 |
+
if not all(isinstance(p, (list, tuple)) and len(p) == 2
|
| 210 |
+
and all(type(v) in (int, float) and math.isfinite(v) for v in p) for p in expected):
|
| 211 |
+
raise ValueError
|
| 212 |
+
if (len(set(points)) < 4 or math.dist(points[0], points[-1]) > 1e-5
|
| 213 |
+
or not all(math.isfinite(lat) and math.isfinite(lon) and abs(lat) <= 90 and abs(lon) <= 180
|
| 214 |
+
and abs(lat - original[1]) <= 5.01e-8 and abs(lon - original[0]) <= 5.01e-8
|
| 215 |
+
for (lat, lon), original in zip(points, expected))):
|
| 216 |
+
raise ValueError
|
| 217 |
+
except (ValueError, TypeError, KeyError, IndexError, ET.ParseError):
|
| 218 |
+
raise RuntimeError("GPX does not match approved source GPS") from None
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def current_design_review(root, label, template):
|
| 222 |
+
item_id = hashlib.sha256(("design:" + label + ":" + template["design_digest"]).encode()).hexdigest()
|
| 223 |
+
with sqlite3.connect((root / "inventory.sqlite3").as_uri() + "?mode=ro", uri=True) as db:
|
| 224 |
+
db.execute("PRAGMA query_only=ON")
|
| 225 |
+
row = db.execute("SELECT status,ai_json FROM items WHERE id=? AND kind='design'", (item_id,)).fetchone()
|
| 226 |
+
return (row[0], json.loads(row[1] or "{}")) if row else ("unknown", {})
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def art_packets(root, probe_id=None, max_routes=None, *, selected_ids=None):
|
| 230 |
+
root = root.resolve()
|
| 231 |
+
rows = approved_art_rows(root)
|
| 232 |
+
if not rows:
|
| 233 |
+
raise RuntimeError("Empty approval list; refusing to synchronize")
|
| 234 |
+
if probe_id:
|
| 235 |
+
rows = [r for r in rows if r["id"] == probe_id]
|
| 236 |
+
if len(rows) != 1:
|
| 237 |
+
raise RuntimeError("Probe ID is not currently approved")
|
| 238 |
+
if selected_ids is not None:
|
| 239 |
+
rows = [r for r in rows if r['id'] in selected_ids]
|
| 240 |
+
if {r['id'] for r in rows} != set(selected_ids):
|
| 241 |
+
raise ApprovalChanged("Selected route approval changed")
|
| 242 |
+
elif max_routes is not None:
|
| 243 |
+
if max_routes <= 0:
|
| 244 |
+
raise RuntimeError("max_routes must be positive")
|
| 245 |
+
rows = rows[:max_routes]
|
| 246 |
+
packets, paths = {}, {}
|
| 247 |
+
finalized = (root / "completion.json").is_file()
|
| 248 |
+
with sqlite3.connect((root / "inventory.sqlite3").as_uri() + "?mode=ro", uri=True) as db:
|
| 249 |
+
db.execute("PRAGMA query_only=ON")
|
| 250 |
+
db.execute("BEGIN")
|
| 251 |
+
for row in rows:
|
| 252 |
+
source_id, label = row["id"], row["label"]
|
| 253 |
+
if (not re.fullmatch(r"[0-9a-f]{64}", source_id) or label not in {"dog", "heart", "tree", "tshirt", "candle", "pants", "star"}
|
| 254 |
+
or row["kind"] != "route" or row["status"] != "ai_pass" or row["geometry_ok"] != 1 or row["seoul_ok"] != 1):
|
| 255 |
+
raise RuntimeError("Approval state changed")
|
| 256 |
+
record_path = Path(row["record_path"])
|
| 257 |
+
record_raw = record_path.read_bytes()
|
| 258 |
+
record = json.loads(record_raw)
|
| 259 |
+
metadata, route = record["metadata"], record["route"]
|
| 260 |
+
design_path = Path(metadata.get("design_file") or ".")
|
| 261 |
+
if not design_path.is_file():
|
| 262 |
+
design_path = root / "designs" / label / (row["design_id"] + ".json")
|
| 263 |
+
design_raw = design_path.read_bytes()
|
| 264 |
+
design = json.loads(design_raw)
|
| 265 |
+
if not isinstance(design, dict):
|
| 266 |
+
raise RuntimeError("Invalid source design snapshot")
|
| 267 |
+
design_digest = hashlib.sha256(canonical(design).encode()).hexdigest()
|
| 268 |
+
if (design_digest != metadata.get("design_digest") or design.get("design_id") != row["design_id"]
|
| 269 |
+
or route.get("design_id") != row["design_id"] or str(design.get("version")) != str(route.get("design_version"))):
|
| 270 |
+
raise RuntimeError("Source design identity changed")
|
| 271 |
+
review = json.loads(row["ai_json"])
|
| 272 |
+
if (review.get("status") != "ai_pass" or review.get("expected_label") != label
|
| 273 |
+
or review.get("blind") is not True or review.get("qwen_performed") is not True
|
| 274 |
+
or not review.get("siglip_support_angles")):
|
| 275 |
+
raise RuntimeError("Missing route review evidence")
|
| 276 |
+
proof = route.get("verification", {})
|
| 277 |
+
if (not all(proof.get(k) is True for k in ("road_only", "graph_connectivity_verified", "access_checked", "directions_checked"))
|
| 278 |
+
or type(proof.get("air_length_m")) not in (int, float) or proof["air_length_m"] != 0
|
| 279 |
+
or record.get("assessment", {}).get("success") is not True
|
| 280 |
+
or record.get("assessment", {}).get("route_valid") is not True):
|
| 281 |
+
raise RuntimeError("Missing road or geometry evidence")
|
| 282 |
+
graph_hash = metadata.get("graph_sha256", "")
|
| 283 |
+
if not re.fullmatch(r"[0-9a-f]{64}", graph_hash):
|
| 284 |
+
raise RuntimeError("Missing graph digest")
|
| 285 |
+
length = route.get("length_m")
|
| 286 |
+
if type(length) not in (int, float) or not math.isfinite(length) or length <= 0:
|
| 287 |
+
raise RuntimeError("Invalid source road length")
|
| 288 |
+
design_item_id = hashlib.sha256(("design:" + label + ":" + design_digest).encode()).hexdigest()
|
| 289 |
+
design_review = db.execute("SELECT status,ai_json FROM items WHERE id=? AND kind='design'", (design_item_id,)).fetchone()
|
| 290 |
+
design_status = design_review[0] if design_review else "unknown"
|
| 291 |
+
design_evidence = json.loads(design_review[1] or "{}") if design_review else {}
|
| 292 |
+
validate_design(root, design, design_status, design_evidence, label)
|
| 293 |
+
gpx_path = root / "gpx" / label / (source_id + ".gpx")
|
| 294 |
+
gpx = gpx_path.read_bytes()
|
| 295 |
+
verify_source_gpx(gpx, route.get("gps", []))
|
| 296 |
+
packet = {"contract_version": 2, "label": label, "region_code": row["city_id"], "region_name": row["city_name"],
|
| 297 |
+
"sha256": hashlib.sha256(gpx).hexdigest(), "accepted_at": row["accepted_at"], "review_status": "ai_pass",
|
| 298 |
+
"ai_protocol": review["protocol"], "geometry_ok": True, "seoul_ok": True, "collection_finalized": finalized,
|
| 299 |
+
"gpx_base64": base64.b64encode(gpx).decode(),
|
| 300 |
+
"template": {"design_id": design["design_id"], "source_version": str(design["version"]),
|
| 301 |
+
"design_digest": design_digest, "snapshot": design, "design_ai_status": design_status,
|
| 302 |
+
"design_ai_evidence": design_evidence},
|
| 303 |
+
"route_ai_evidence": review, "road_verification": proof, "graph_sha256": graph_hash,
|
| 304 |
+
"distance_m": max(1, int(math.floor(length + .5))), "validator_version": "lab-approved-gpx-import-v1"}
|
| 305 |
+
packets[source_id] = packet
|
| 306 |
+
validate_packet(packet)
|
| 307 |
+
paths[source_id] = {str(p): hashlib.sha256(raw).hexdigest() for p, raw in
|
| 308 |
+
[(record_path, record_raw), (design_path, design_raw), (gpx_path, gpx)]}
|
| 309 |
+
return rows, packets, paths
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def validate_design(root, design, status, evidence, label):
|
| 314 |
+
"""Keep design classification independent from the accepted route."""
|
| 315 |
+
points = design.get('points')
|
| 316 |
+
if (status not in {'ai_pass', 'ai_reject', 'ai_review', 'unknown'}
|
| 317 |
+
or not isinstance(evidence, dict) or evidence.get('status', status) != status
|
| 318 |
+
or (status == 'ai_pass' and not approval_validator(root)(canonical(evidence), label))
|
| 319 |
+
or not isinstance(design.get('name'), str) or not 0 < len(design['name']) <= 200
|
| 320 |
+
or not isinstance(design.get('outline'), dict)
|
| 321 |
+
or design['outline'].get('coordinate_system') != 'cartesian_y_up'
|
| 322 |
+
or not isinstance(points, list) or not 3 <= len(points) <= 10000
|
| 323 |
+
or not all(isinstance(p, list) and len(p) == 2
|
| 324 |
+
and all(type(v) in (int, float) and math.isfinite(v) for v in p) for p in points)
|
| 325 |
+
or len(set(map(tuple, points))) < 3 or len(canonical(design).encode()) > 512 * 1024):
|
| 326 |
+
raise RuntimeError("Invalid independent design evidence or snapshot")
|
| 327 |
+
extent = max(max(p[j] for p in points) - min(p[j] for p in points) for j in (0, 1))
|
| 328 |
+
if not math.isfinite(extent) or extent <= 0:
|
| 329 |
+
raise RuntimeError("Invalid source design extent")
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def validate_packet(packet):
|
| 333 |
+
template = packet['template']
|
| 334 |
+
accepted = datetime.fromisoformat(packet['accepted_at'].replace('Z', '+00:00'))
|
| 335 |
+
if (accepted.tzinfo is None
|
| 336 |
+
or not re.fullmatch(r'[a-z0-9][a-z0-9_-]{0,95}', template['design_id'])
|
| 337 |
+
or not 1 <= len(template['source_version']) <= 100
|
| 338 |
+
or not isinstance(packet['region_code'], str) or not 1 <= len(packet['region_code']) <= 40
|
| 339 |
+
or not isinstance(packet['region_name'], str) or not 1 <= len(packet['region_name']) <= 200
|
| 340 |
+
or not 1 <= len(packet['ai_protocol']) <= 200
|
| 341 |
+
or not 0 < packet['distance_m'] < 2147483648
|
| 342 |
+
or len(canonical(packet).encode()) > 2 * 1024 * 1024):
|
| 343 |
+
raise RuntimeError("Invalid v2 route metadata")
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def normalized_endpoint(destination):
|
| 347 |
+
parts = urllib.parse.urlsplit(destination)
|
| 348 |
+
if (parts.username or parts.password or parts.query or parts.fragment or not parts.hostname
|
| 349 |
+
or (parts.scheme != 'https' and not (parts.scheme == 'http' and parts.hostname in {'localhost', '127.0.0.1', '::1'}))):
|
| 350 |
+
raise RuntimeError("Use an HTTPS API URL without credentials, query or fragment")
|
| 351 |
+
return destination.rstrip('/')
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def ledger_ids(state):
|
| 355 |
+
"""v1 history counts toward the existing cap, but never proves v2 storage."""
|
| 356 |
+
ids = {row[0] for row in state.execute('SELECT id FROM sent')}
|
| 357 |
+
if state.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='http_v2_sent'").fetchone():
|
| 358 |
+
ids.update(row[0] for row in state.execute('SELECT id FROM http_v2_sent'))
|
| 359 |
+
return ids
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def select_v2_rows(root, state, probe_id=None, max_routes=None):
|
| 363 |
+
if max_routes is not None and (max_routes < 1 or probe_id is not None):
|
| 364 |
+
raise ValueError("max_routes must be positive and cannot be combined with probe_id")
|
| 365 |
+
rows = approved_art_rows(root)
|
| 366 |
+
if not rows:
|
| 367 |
+
raise RuntimeError("Empty approval list; refusing to synchronize")
|
| 368 |
+
total = len(rows)
|
| 369 |
+
if probe_id is not None:
|
| 370 |
+
rows = [r for r in rows if r['id'] == probe_id]
|
| 371 |
+
if len(rows) != 1:
|
| 372 |
+
raise RuntimeError("Probe ID is not currently approved")
|
| 373 |
+
if max_routes is not None:
|
| 374 |
+
sent_ids = ledger_ids(state)
|
| 375 |
+
if sent_ids - {row['id'] for row in rows}:
|
| 376 |
+
raise RuntimeError("Previously sent routes are no longer approved; refusing a limited synchronization")
|
| 377 |
+
if len(sent_ids) > max_routes:
|
| 378 |
+
raise RuntimeError("Previously sent route count already exceeds max_routes")
|
| 379 |
+
retained = [r for r in rows if r['id'] in sent_ids]
|
| 380 |
+
extra = [r for r in rows if r['id'] not in sent_ids]
|
| 381 |
+
rows = sorted(retained + extra[:max_routes - len(retained)], key=lambda r: r['id'])
|
| 382 |
+
return total, {row['id'] for row in rows}
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def check_only_v2(root, state_path, destination, probe_id=None, max_routes=None):
|
| 386 |
+
endpoint = normalized_endpoint(destination)
|
| 387 |
+
# mode=ro: no schema creation, lock acquisition, credential read or HTTP.
|
| 388 |
+
with sqlite3.connect(state_path.resolve().as_uri() + '?mode=ro', uri=True) as state:
|
| 389 |
+
state.execute('PRAGMA query_only=ON')
|
| 390 |
+
state.execute('BEGIN')
|
| 391 |
+
total, selected = select_v2_rows(root, state, probe_id, max_routes)
|
| 392 |
+
rows, packets, _ = art_packets(root, selected_ids=selected)
|
| 393 |
+
receipts = {}
|
| 394 |
+
if state.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='http_v2_sent'").fetchone():
|
| 395 |
+
columns = {r[1] for r in state.execute('PRAGMA table_info(http_v2_sent)')}
|
| 396 |
+
if 'endpoint' in columns:
|
| 397 |
+
receipts = dict(state.execute('SELECT id,signature FROM http_v2_sent WHERE endpoint=?', (endpoint,)))
|
| 398 |
+
confirmed = sum(receipts.get(k) == content_sha(v) for k, v in packets.items())
|
| 399 |
+
return {'contract_version': 2, 'approved': total, 'selected': len(rows),
|
| 400 |
+
'templates': len({(p['label'], p['template']['design_digest']) for p in packets.values()}),
|
| 401 |
+
'v1_history': state.execute('SELECT count(*) FROM sent').fetchone()[0],
|
| 402 |
+
'v2_confirmed': confirmed, 'v2_pending': len(rows) - confirmed,
|
| 403 |
+
'http_requests': 0, 'credential_reads': 0, 'state_writes': 0}
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def request_v2(url, token, method, payload, *, before_attempt=None):
|
| 407 |
+
"""Bounded retries retain the session request ID and the exact route payload."""
|
| 408 |
+
for attempt in range(3):
|
| 409 |
+
if before_attempt is not None:
|
| 410 |
+
before_attempt()
|
| 411 |
+
try:
|
| 412 |
+
reply = send(url, token, method, payload)
|
| 413 |
+
if not isinstance(reply, dict):
|
| 414 |
+
raise ReceiverUnhealthy("Invalid v2 JSON response")
|
| 415 |
+
return reply
|
| 416 |
+
except urllib.error.HTTPError as exc:
|
| 417 |
+
if exc.code not in {408, 429, 500, 502, 503, 504} or attempt == 2:
|
| 418 |
+
raise
|
| 419 |
+
except (urllib.error.URLError, TimeoutError, ConnectionError):
|
| 420 |
+
if attempt == 2:
|
| 421 |
+
raise
|
| 422 |
+
time.sleep(2 ** attempt)
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def validate_session(reply, expected_generation, mode, manifest_hash, session_id=None):
|
| 426 |
+
try:
|
| 427 |
+
received_id = str(uuid.UUID(reply['session_id']))
|
| 428 |
+
if uuid.UUID(received_id).int == 0:
|
| 429 |
+
raise ValueError
|
| 430 |
+
except (KeyError, ValueError, TypeError, AttributeError):
|
| 431 |
+
raise ReceiverUnhealthy("Invalid sync session response") from None
|
| 432 |
+
if (type(reply.get('contract_version')) is not int or reply['contract_version'] != 2
|
| 433 |
+
or type(reply.get('generation')) is not int or reply['generation'] != expected_generation
|
| 434 |
+
or reply.get('mode') != mode or reply.get('manifest_sha256') != manifest_hash
|
| 435 |
+
or reply.get('session_status') not in {'open', 'completed'}
|
| 436 |
+
or (session_id is not None and received_id != session_id)):
|
| 437 |
+
raise ReceiverUnhealthy("Inconsistent or stale sync session response")
|
| 438 |
+
return received_id
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def validate_receipt(reply, source_id, packet, previous=None):
|
| 442 |
+
try:
|
| 443 |
+
valid_ids = all(isinstance(reply[k], str) and str(uuid.UUID(reply[k])) == reply[k]
|
| 444 |
+
and uuid.UUID(reply[k]).int != 0 for k in ('art_template_id', 'art_route_candidate_id'))
|
| 445 |
+
except (KeyError, ValueError, TypeError, AttributeError):
|
| 446 |
+
valid_ids = False
|
| 447 |
+
if (not valid_ids or type(reply.get('contract_version')) is not int or reply['contract_version'] != 2
|
| 448 |
+
or reply.get('status') != 'stored' or reply.get('result') not in {'created', 'updated', 'unchanged'}
|
| 449 |
+
or reply.get('source_id') != source_id or reply.get('sha256') != packet['sha256']
|
| 450 |
+
or reply.get('payload_sha256') != content_sha(packet)
|
| 451 |
+
or reply.get('design_ai_status') != packet['template']['design_ai_status']
|
| 452 |
+
or reply.get('route_validation_status') != 'passed'):
|
| 453 |
+
raise ReceiverUnhealthy("Missing or inconsistent art storage receipt")
|
| 454 |
+
if previous and (reply['art_route_candidate_id'] != previous['candidate_id']
|
| 455 |
+
or (previous['signature'] == content_sha(packet) and reply['art_template_id'] != previous['template_id'])):
|
| 456 |
+
raise ReceiverUnhealthy("Stored art identifiers changed unexpectedly")
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def initialize_v2_state(state):
|
| 460 |
+
state.execute('SELECT id,signature FROM sent LIMIT 0')
|
| 461 |
+
state.execute('''CREATE TABLE IF NOT EXISTS http_v2_runs (
|
| 462 |
+
client_request_id TEXT PRIMARY KEY, signature TEXT NOT NULL, expected_generation INTEGER NOT NULL,
|
| 463 |
+
session_id TEXT, generation INTEGER, completed INTEGER NOT NULL DEFAULT 0)''')
|
| 464 |
+
state.execute('''CREATE TABLE IF NOT EXISTS http_v2_sent (
|
| 465 |
+
id TEXT PRIMARY KEY, signature TEXT NOT NULL, session_id TEXT NOT NULL,
|
| 466 |
+
template_id TEXT NOT NULL, candidate_id TEXT NOT NULL, endpoint TEXT NOT NULL DEFAULT '')''')
|
| 467 |
+
columns = {r[1] for r in state.execute('PRAGMA table_info(http_v2_sent)')}
|
| 468 |
+
if 'endpoint' not in columns:
|
| 469 |
+
# Old patch receipts have no destination binding; never reuse them blindly.
|
| 470 |
+
state.execute("ALTER TABLE http_v2_sent ADD COLUMN endpoint TEXT NOT NULL DEFAULT ''")
|
| 471 |
+
state.commit()
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def sync_v2(root, destination, token, state_path, probe_id=None, max_routes=None,
|
| 475 |
+
allow_updates=False, allow_reapproval=False, new_session=False):
|
| 476 |
+
root = root.resolve()
|
| 477 |
+
endpoint = normalized_endpoint(destination)
|
| 478 |
+
# Never replace the installation's original ledger with a new empty database.
|
| 479 |
+
state = sqlite3.connect(state_path.resolve().as_uri() + '?mode=rw', uri=True)
|
| 480 |
+
state.row_factory = sqlite3.Row
|
| 481 |
+
try:
|
| 482 |
+
total, selected = select_v2_rows(root, state, probe_id, max_routes)
|
| 483 |
+
rows, packets, paths = art_packets(root, selected_ids=selected)
|
| 484 |
+
manifest = {key: content_sha(packet) for key, packet in packets.items()}
|
| 485 |
+
manifest_hash = hashlib.sha256(canonical(manifest).encode()).hexdigest()
|
| 486 |
+
mode = 'partial' if probe_id is not None or max_routes is not None else 'full'
|
| 487 |
+
signature = hashlib.sha256(canonical([endpoint, str(root), manifest_hash, mode,
|
| 488 |
+
allow_updates, allow_reapproval]).encode()).hexdigest()
|
| 489 |
+
|
| 490 |
+
def assert_route(row):
|
| 491 |
+
key = row['id']
|
| 492 |
+
if (approved_art_rows(root, key) != [row]
|
| 493 |
+
or any(hashlib.sha256(Path(p).read_bytes()).hexdigest() != h for p, h in paths[key].items())
|
| 494 |
+
or current_design_review(root, row['label'], packets[key]['template']) != (
|
| 495 |
+
packets[key]['template']['design_ai_status'], packets[key]['template']['design_ai_evidence'])):
|
| 496 |
+
raise ApprovalChanged("Approval or source changed; resume from a new snapshot")
|
| 497 |
+
if packets[key]['collection_finalized'] != (root / 'completion.json').is_file():
|
| 498 |
+
raise ApprovalChanged("Collection finalization changed")
|
| 499 |
+
|
| 500 |
+
def assert_manifest():
|
| 501 |
+
if mode == 'full' and approved_art_rows(root) != rows:
|
| 502 |
+
raise ApprovalChanged("Approval list changed; snapshot withheld")
|
| 503 |
+
for row in rows:
|
| 504 |
+
assert_route(row)
|
| 505 |
+
|
| 506 |
+
# All source validation precedes local schema changes and the first HTTP request.
|
| 507 |
+
initialize_v2_state(state)
|
| 508 |
+
run = None if new_session else state.execute(
|
| 509 |
+
'SELECT * FROM http_v2_runs WHERE signature=? AND completed=0 ORDER BY rowid DESC LIMIT 1', (signature,)).fetchone()
|
| 510 |
+
if run is None:
|
| 511 |
+
remote = request_v2(endpoint + '/sync-state', token, 'GET', None, before_attempt=assert_manifest)
|
| 512 |
+
if (type(remote.get('contract_version')) is not int or remote['contract_version'] != 2
|
| 513 |
+
or type(remote.get('generation')) is not int or remote['generation'] < 0):
|
| 514 |
+
raise ReceiverUnhealthy("Receiver does not support fenced v2 imports")
|
| 515 |
+
request_id = str(uuid.uuid4())
|
| 516 |
+
if new_session:
|
| 517 |
+
# Do not rediscover an older superseded checkpoint after this run completes.
|
| 518 |
+
state.execute('UPDATE http_v2_runs SET completed=2 WHERE signature=? AND completed=0', (signature,))
|
| 519 |
+
state.execute('INSERT INTO http_v2_runs(client_request_id,signature,expected_generation) VALUES (?,?,?)',
|
| 520 |
+
(request_id, signature, remote['generation']))
|
| 521 |
+
state.commit() # Persist before POST; a lost response reuses the same request ID.
|
| 522 |
+
run = state.execute('SELECT * FROM http_v2_runs WHERE client_request_id=?', (request_id,)).fetchone()
|
| 523 |
+
session = request_v2(endpoint + '/sync-sessions', token, 'POST', {'contract_version': 2,
|
| 524 |
+
'client_request_id': run['client_request_id'], 'expected_generation': run['expected_generation'],
|
| 525 |
+
'mode': mode, 'manifest': manifest, 'manifest_sha256': manifest_hash}, before_attempt=assert_manifest)
|
| 526 |
+
generation = run['expected_generation'] + 1
|
| 527 |
+
session_id = validate_session(session, generation, mode, manifest_hash, run['session_id'])
|
| 528 |
+
current = request_v2(endpoint + '/sync-sessions/' + session_id, token, 'GET', None, before_attempt=assert_manifest)
|
| 529 |
+
validate_session(current, generation, mode, manifest_hash, session_id)
|
| 530 |
+
state.execute('UPDATE http_v2_runs SET session_id=?,generation=? WHERE client_request_id=?',
|
| 531 |
+
(session_id, generation, run['client_request_id']))
|
| 532 |
+
state.commit()
|
| 533 |
+
uploaded = 0
|
| 534 |
+
for row in rows:
|
| 535 |
+
key = row['id']
|
| 536 |
+
assert_route(row)
|
| 537 |
+
previous = state.execute('SELECT * FROM http_v2_sent WHERE id=? AND endpoint=?', (key, endpoint)).fetchone()
|
| 538 |
+
packet = {**packets[key], 'session_id': session_id, 'generation': generation, 'payload_sha256': manifest[key]}
|
| 539 |
+
if previous and previous['session_id'] == session_id and previous['signature'] == manifest[key]:
|
| 540 |
+
# Validate persisted receipt fields before trusting a resumable checkpoint.
|
| 541 |
+
validate_receipt({'contract_version': 2, 'status': 'stored', 'result': 'unchanged',
|
| 542 |
+
'source_id': key, 'sha256': packet['sha256'], 'payload_sha256': manifest[key],
|
| 543 |
+
'design_ai_status': packet['template']['design_ai_status'], 'route_validation_status': 'passed',
|
| 544 |
+
'art_template_id': previous['template_id'], 'art_route_candidate_id': previous['candidate_id']}, key, packet)
|
| 545 |
+
continue
|
| 546 |
+
if previous and allow_updates:
|
| 547 |
+
packet['previous_payload_sha256'] = previous['signature']
|
| 548 |
+
if allow_reapproval:
|
| 549 |
+
packet['reapprove'] = True
|
| 550 |
+
receipt = request_v2(endpoint + '/routes/' + key, token, 'PUT', packet,
|
| 551 |
+
before_attempt=lambda row=row: assert_route(row))
|
| 552 |
+
validate_receipt(receipt, key, packet, previous)
|
| 553 |
+
state.execute('''INSERT INTO http_v2_sent(id,signature,session_id,template_id,candidate_id,endpoint)
|
| 554 |
+
VALUES (?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET signature=excluded.signature,
|
| 555 |
+
session_id=excluded.session_id,template_id=excluded.template_id,
|
| 556 |
+
candidate_id=excluded.candidate_id,endpoint=excluded.endpoint''',
|
| 557 |
+
(key, manifest[key], session_id, receipt['art_template_id'], receipt['art_route_candidate_id'], endpoint))
|
| 558 |
+
state.commit()
|
| 559 |
+
uploaded += 1
|
| 560 |
+
assert_manifest()
|
| 561 |
+
current = request_v2(endpoint + '/sync-sessions/' + session_id, token, 'GET', None, before_attempt=assert_manifest)
|
| 562 |
+
validate_session(current, generation, mode, manifest_hash, session_id)
|
| 563 |
+
if mode == 'full':
|
| 564 |
+
ids = sorted(manifest)
|
| 565 |
+
result = request_v2(endpoint + '/approval-snapshot', token, 'POST', {'contract_version': 2,
|
| 566 |
+
'session_id': session_id, 'generation': generation, 'manifest_sha256': manifest_hash,
|
| 567 |
+
'ids': ids, 'count': len(ids), 'ids_sha256': hashlib.sha256('\n'.join(ids).encode()).hexdigest(),
|
| 568 |
+
'collection_finalized': packets[ids[0]]['collection_finalized']}, before_attempt=assert_manifest)
|
| 569 |
+
if (type(result.get('active_count')) is not int or result['active_count'] != len(ids)
|
| 570 |
+
or type(result.get('deactivated_count')) is not int or result['deactivated_count'] < 0):
|
| 571 |
+
raise ReceiverUnhealthy("Invalid approval snapshot response")
|
| 572 |
+
else:
|
| 573 |
+
result = request_v2(endpoint + '/sync-sessions/' + session_id + '/complete?generation=' + str(generation),
|
| 574 |
+
token, 'POST', {}, before_attempt=assert_manifest)
|
| 575 |
+
validate_session(result, generation, mode, manifest_hash, session_id)
|
| 576 |
+
if (result.get('session_status') != 'completed' or type(result.get('deactivated_count')) is not int
|
| 577 |
+
or result['deactivated_count'] != 0):
|
| 578 |
+
raise ReceiverUnhealthy("Invalid partial session completion response")
|
| 579 |
+
if (type(result.get('contract_version')) is not int or result['contract_version'] != 2
|
| 580 |
+
or result.get('session_id') != session_id or type(result.get('generation')) is not int
|
| 581 |
+
or result['generation'] != generation):
|
| 582 |
+
raise ReceiverUnhealthy("Inconsistent sync completion response")
|
| 583 |
+
state.execute('UPDATE http_v2_runs SET completed=1 WHERE client_request_id=?', (run['client_request_id'],))
|
| 584 |
+
state.commit()
|
| 585 |
+
return {'contract_version': 2, 'approved': total, 'selected': len(rows), 'uploaded': uploaded,
|
| 586 |
+
'sent_total': len(ledger_ids(state)), 'snapshot_sent': mode == 'full',
|
| 587 |
+
'generation': generation, 'deactivated_count': result['deactivated_count']}
|
| 588 |
+
finally:
|
| 589 |
+
state.close()
|
| 590 |
+
if state_path.exists():
|
| 591 |
+
os.chmod(state_path, 0o600)
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def sync(root: Path, destination: str, token: str, state_path: Path,
|
| 595 |
+
probe_id: str | None = None, max_routes: int | None = None) -> dict:
|
| 596 |
+
if max_routes is not None and (max_routes < 1 or probe_id is not None):
|
| 597 |
+
raise ValueError("max_routes must be positive and cannot be combined with probe_id")
|
| 598 |
+
rows = approved_rows(root)
|
| 599 |
+
if not rows:
|
| 600 |
+
raise RuntimeError("Empty approval list; refusing to synchronize")
|
| 601 |
+
approved_count = len(rows)
|
| 602 |
+
if probe_id is not None:
|
| 603 |
+
rows = [row for row in rows if row["id"] == probe_id]
|
| 604 |
+
if len(rows) != 1:
|
| 605 |
+
raise RuntimeError("Probe ID is not in the current approval list")
|
| 606 |
+
# mode=rw refuses to silently create an empty replacement ledger.
|
| 607 |
+
state = sqlite3.connect(state_path.resolve().as_uri()+'?mode=rw', uri=True)
|
| 608 |
+
try:
|
| 609 |
+
state.execute("SELECT id,signature FROM sent LIMIT 0")
|
| 610 |
+
if max_routes is not None:
|
| 611 |
+
sent_ids = {row[0] for row in state.execute("SELECT id FROM sent")}
|
| 612 |
+
approved_ids = {row["id"] for row in rows}
|
| 613 |
+
if sent_ids - approved_ids:
|
| 614 |
+
raise RuntimeError("Previously sent routes are no longer approved; refusing a limited synchronization")
|
| 615 |
+
if len(sent_ids) > max_routes:
|
| 616 |
+
raise RuntimeError("Previously sent route count already exceeds max_routes")
|
| 617 |
+
retained = [row for row in rows if row["id"] in sent_ids]
|
| 618 |
+
additional = [row for row in rows if row["id"] not in sent_ids]
|
| 619 |
+
rows = sorted(retained + additional[:max_routes - len(retained)], key=lambda row: row["id"])
|
| 620 |
+
ids = []
|
| 621 |
+
file_hashes = {}
|
| 622 |
+
uploaded = 0
|
| 623 |
+
finalized = (root/'completion.json').is_file()
|
| 624 |
+
# Validate every file before the first PUT; partial preflight is unsafe.
|
| 625 |
+
prepared = [(row, *prepare_route(root, row, finalized)) for row in rows]
|
| 626 |
+
for row, gpx, metadata, signature in prepared:
|
| 627 |
+
source_id, label = row["id"], row["label"]
|
| 628 |
+
file_hashes[source_id] = metadata['sha256']
|
| 629 |
+
previous = state.execute("SELECT signature FROM sent WHERE id=?", (source_id,)).fetchone()
|
| 630 |
+
if probe_id is not None or not previous or previous[0] != signature:
|
| 631 |
+
current = approved_rows(root, source_id)
|
| 632 |
+
if len(current) != 1 or tuple(current[0]) != tuple(row):
|
| 633 |
+
raise ApprovalChanged("Approval changed before upload; retry from a new snapshot")
|
| 634 |
+
if hashlib.sha256((root/'gpx'/label/(source_id+'.gpx')).read_bytes()).hexdigest() != metadata['sha256']:
|
| 635 |
+
raise RuntimeError("GPX changed before upload")
|
| 636 |
+
received = send(destination.rstrip("/") + "/routes/" + source_id, token, "PUT",
|
| 637 |
+
{**metadata, "gpx_base64": base64.b64encode(gpx).decode()})
|
| 638 |
+
if (not isinstance(received, dict) or received.get('status') != 'stored'
|
| 639 |
+
or received.get('source_id') != source_id or received.get('sha256') != metadata['sha256']):
|
| 640 |
+
raise ReceiverUnhealthy('Unexpected route storage response')
|
| 641 |
+
state.execute("INSERT INTO sent VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET signature=excluded.signature",
|
| 642 |
+
(source_id, signature))
|
| 643 |
+
state.commit()
|
| 644 |
+
uploaded += 1
|
| 645 |
+
ids.append(source_id)
|
| 646 |
+
if probe_id is not None:
|
| 647 |
+
return {"probe_id": probe_id, "uploaded": uploaded, "snapshot_sent": False}
|
| 648 |
+
if max_routes is not None:
|
| 649 |
+
return {"approved": approved_count, "selected": len(rows), "uploaded": uploaded,
|
| 650 |
+
"sent_total": state.execute("SELECT count(*) FROM sent").fetchone()[0],
|
| 651 |
+
"max_routes": max_routes, "snapshot_sent": False}
|
| 652 |
+
# Do not revoke from a partial file set or a changing DB approval list.
|
| 653 |
+
if [tuple(row) for row in rows] != [tuple(row) for row in approved_rows(root)]:
|
| 654 |
+
raise ApprovalChanged("Approval list changed during upload; retry from a new snapshot")
|
| 655 |
+
if finalized != (root/'completion.json').is_file():
|
| 656 |
+
raise ApprovalChanged("Collection finalization changed during upload; retry from a new snapshot")
|
| 657 |
+
if any(hashlib.sha256((root / "gpx" / row["label"] / (row["id"] + ".gpx")).read_bytes()).hexdigest()
|
| 658 |
+
!= file_hashes[row["id"]] for row in rows):
|
| 659 |
+
raise RuntimeError("GPX files changed during upload; retry from a new snapshot")
|
| 660 |
+
digest = hashlib.sha256("\n".join(ids).encode()).hexdigest()
|
| 661 |
+
result = send(destination.rstrip("/") + "/approval-snapshot", token, "POST", {
|
| 662 |
+
"ids": ids, "count": len(ids), "ids_sha256": digest,
|
| 663 |
+
"collection_finalized": finalized,
|
| 664 |
+
})
|
| 665 |
+
if (not isinstance(result, dict) or type(result.get('active_count')) is not int
|
| 666 |
+
or result['active_count'] != len(ids) or type(result.get('deactivated_count')) is not int
|
| 667 |
+
or result['deactivated_count'] < 0):
|
| 668 |
+
raise ReceiverUnhealthy('Unexpected approval snapshot response')
|
| 669 |
+
return {"approved": len(ids), "uploaded": uploaded, "snapshot_sent": True,
|
| 670 |
+
"active_count": result['active_count'], "deactivated_count": result['deactivated_count'], "error": False}
|
| 671 |
+
finally:
|
| 672 |
+
state.close()
|
| 673 |
+
if state_path.exists():
|
| 674 |
+
os.chmod(state_path, 0o600)
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def main() -> None:
|
| 678 |
+
parser = argparse.ArgumentParser()
|
| 679 |
+
parser.add_argument("--root", type=Path, required=True)
|
| 680 |
+
parser.add_argument("--url", required=True, help="Full /api/v1/ai/internal/route-library URL")
|
| 681 |
+
parser.add_argument("--token-file", type=Path, required=True)
|
| 682 |
+
parser.add_argument("--state", type=Path, default=DEFAULT_STATE)
|
| 683 |
+
parser.add_argument("--check-only", action="store_true", help="Read-only preflight; never reads the token or sends HTTP")
|
| 684 |
+
parser.add_argument("--workers", type=int, choices=[1], default=1, help="One sequential upload worker only")
|
| 685 |
+
parser.add_argument("--initial-delay-seconds", type=int, default=0)
|
| 686 |
+
parser.add_argument("--interval-seconds", type=int, default=0, help="Repeat after this delay; 0 runs once")
|
| 687 |
+
parser.add_argument("--probe-id", help="Send one approved GPX without an approval snapshot")
|
| 688 |
+
parser.add_argument("--max-routes", type=int,
|
| 689 |
+
help="Cap total sent routes, including previous sends; never send an approval snapshot")
|
| 690 |
+
parser.add_argument("--contract-version", type=int, choices=[1, 2], default=1,
|
| 691 |
+
help="Explicitly opt in to v2 art storage; existing startup stays on v1")
|
| 692 |
+
parser.add_argument("--allow-updates", action="store_true", help="v2 compare-and-swap using a verified previous receipt")
|
| 693 |
+
parser.add_argument("--allow-reapproval", action="store_true", help="v2 explicit reapproval with a later source approval")
|
| 694 |
+
parser.add_argument("--new-session", action="store_true", help="v2 explicitly replace a pending session")
|
| 695 |
+
args = parser.parse_args()
|
| 696 |
+
if args.interval_seconds < 0 or args.initial_delay_seconds < 0:
|
| 697 |
+
parser.error("--interval-seconds must be nonnegative")
|
| 698 |
+
if args.probe_id and args.interval_seconds:
|
| 699 |
+
parser.error("--probe-id cannot be used with --interval-seconds")
|
| 700 |
+
if args.max_routes is not None and (args.max_routes < 1 or args.probe_id or args.interval_seconds):
|
| 701 |
+
parser.error("--max-routes must be positive and cannot be used with --probe-id or --interval-seconds")
|
| 702 |
+
if args.contract_version != 2 and (args.allow_updates or args.allow_reapproval or args.new_session):
|
| 703 |
+
parser.error("Session and update options require --contract-version 2")
|
| 704 |
+
if not args.state.is_file():
|
| 705 |
+
parser.error("Existing state.sqlite3 is required; refusing to create a replacement")
|
| 706 |
+
if args.state.resolve() != DEFAULT_STATE.resolve():
|
| 707 |
+
parser.error("This installation must reuse the original DemoService state.sqlite3")
|
| 708 |
+
# A preflight must not rewrite even the live lock or status file.
|
| 709 |
+
if args.check_only:
|
| 710 |
+
try:
|
| 711 |
+
result = (check_only_v2(args.root, args.state, args.url, args.probe_id, args.max_routes)
|
| 712 |
+
if args.contract_version == 2 else check_only(args.root, args.state))
|
| 713 |
+
except (OSError, ValueError, RuntimeError, sqlite3.Error, KeyError, TypeError, AttributeError):
|
| 714 |
+
raise SystemExit("Local preflight failed; input details suppressed") from None
|
| 715 |
+
print(json.dumps(result, ensure_ascii=False))
|
| 716 |
+
return
|
| 717 |
+
try:
|
| 718 |
+
with single_instance():
|
| 719 |
+
if HALT_PATH.exists():
|
| 720 |
+
raise SystemExit('Transmission is halted; resolve HALTED.json before resuming')
|
| 721 |
+
token = args.token_file.read_text(encoding="utf-8").strip()
|
| 722 |
+
if not token:
|
| 723 |
+
raise SystemExit("Empty receiver token")
|
| 724 |
+
def stopped(signum, _frame):
|
| 725 |
+
write_status(STATUS_PATH, {'state':'stopped','error':False,'signal':signum,'workers':1})
|
| 726 |
+
raise SystemExit(0)
|
| 727 |
+
signal.signal(signal.SIGTERM, stopped)
|
| 728 |
+
signal.signal(signal.SIGINT, stopped)
|
| 729 |
+
if args.initial_delay_seconds:
|
| 730 |
+
write_status(STATUS_PATH, {'state':'sleeping','error':False,'workers':1,
|
| 731 |
+
'next_delay_seconds':args.initial_delay_seconds})
|
| 732 |
+
time.sleep(args.initial_delay_seconds)
|
| 733 |
+
while True:
|
| 734 |
+
try:
|
| 735 |
+
check_health(args.url)
|
| 736 |
+
write_status(STATUS_PATH, {'state':'transferring','error':False,'workers':1})
|
| 737 |
+
if args.contract_version == 2:
|
| 738 |
+
result = sync_v2(args.root, args.url, token, args.state, args.probe_id, args.max_routes,
|
| 739 |
+
args.allow_updates, args.allow_reapproval, args.new_session)
|
| 740 |
+
args.new_session = False
|
| 741 |
+
else:
|
| 742 |
+
result = sync(args.root, args.url, token, args.state, args.probe_id, args.max_routes)
|
| 743 |
+
check_health(args.url)
|
| 744 |
+
write_status(STATUS_PATH, {'state':'sleeping' if args.interval_seconds else 'complete',
|
| 745 |
+
'workers':1,'interval_seconds':args.interval_seconds,**result})
|
| 746 |
+
print(json.dumps(result, ensure_ascii=False), flush=True)
|
| 747 |
+
except ApprovalChanged:
|
| 748 |
+
write_status(STATUS_PATH, {'state':'approval_changed','error':False,'workers':1})
|
| 749 |
+
if not args.interval_seconds:
|
| 750 |
+
raise SystemExit('Approval changed during synchronization; completed uploads are retained') from None
|
| 751 |
+
except (OSError, ValueError, RuntimeError, sqlite3.Error, urllib.error.URLError,
|
| 752 |
+
KeyError, TypeError, AttributeError, OverflowError) as exc:
|
| 753 |
+
error = {'state':'halted','error':True,'error_type':type(exc).__name__,
|
| 754 |
+
'http_status':exc.code if isinstance(exc,urllib.error.HTTPError) else None,'workers':1}
|
| 755 |
+
write_status(HALT_PATH,error)
|
| 756 |
+
write_status(STATUS_PATH,error)
|
| 757 |
+
print(json.dumps(error),flush=True)
|
| 758 |
+
raise SystemExit(1) from None
|
| 759 |
+
if not args.interval_seconds:
|
| 760 |
+
break
|
| 761 |
+
time.sleep(args.interval_seconds)
|
| 762 |
+
except AlreadyRunning:
|
| 763 |
+
raise SystemExit("Another GPX transfer is already running") from None
|
| 764 |
+
|
| 765 |
+
|
| 766 |
+
if __name__ == "__main__":
|
| 767 |
+
main()
|
tools/lab_sync/tests/fixtures/approval_guards.sql
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- Version 1. Installed transactionally; never changes the transmission ledger.
|
| 2 |
+
CREATE TABLE IF NOT EXISTS approval_revocations (
|
| 3 |
+
item_id TEXT PRIMARY KEY,
|
| 4 |
+
label TEXT NOT NULL,
|
| 5 |
+
revoked_at TEXT NOT NULL,
|
| 6 |
+
reason TEXT NOT NULL,
|
| 7 |
+
previous_accepted_at TEXT
|
| 8 |
+
);
|
| 9 |
+
|
| 10 |
+
-- Model classification remains the existing reviewer's responsibility. These
|
| 11 |
+
-- structural gates also reject missing, malformed, or withdrawn review evidence.
|
| 12 |
+
CREATE VIEW IF NOT EXISTS eligible_approval_items AS
|
| 13 |
+
SELECT i.id, i.label, i.city_id, i.design_id FROM items AS i
|
| 14 |
+
WHERE i.kind = 'route' AND i.status = 'ai_pass'
|
| 15 |
+
AND i.geometry_ok = 1 AND i.seoul_ok = 1
|
| 16 |
+
AND i.label IN ('dog','heart','tree','tshirt','candle','pants','star')
|
| 17 |
+
AND CASE WHEN json_valid(i.ai_json) THEN COALESCE(
|
| 18 |
+
json_type(i.ai_json) = 'object'
|
| 19 |
+
AND json_extract(i.ai_json, '$.status') = 'ai_pass'
|
| 20 |
+
AND json_extract(i.ai_json, '$.expected_label') = i.label
|
| 21 |
+
AND json_type(i.ai_json, '$.blind') = 'true'
|
| 22 |
+
AND json_type(i.ai_json, '$.qwen_performed') = 'true'
|
| 23 |
+
AND json_type(i.ai_json, '$.protocol') = 'text'
|
| 24 |
+
AND length(trim(json_extract(i.ai_json, '$.protocol'))) > 0
|
| 25 |
+
AND json_type(i.ai_json, '$.siglip_support_angles') = 'array'
|
| 26 |
+
AND json_array_length(i.ai_json, '$.siglip_support_angles') > 0
|
| 27 |
+
AND json_type(i.ai_json, '$.siglip_views') = 'array'
|
| 28 |
+
AND EXISTS (SELECT 1 FROM json_each(i.ai_json, '$.siglip_views') AS s
|
| 29 |
+
WHERE CASE WHEN s.type = 'object'
|
| 30 |
+
THEN json_extract(s.value, '$.top_label') = i.label ELSE 0 END)
|
| 31 |
+
AND json_type(i.ai_json, '$.qwen') = 'array'
|
| 32 |
+
AND json_array_length(i.ai_json, '$.qwen') > 0
|
| 33 |
+
AND NOT EXISTS (SELECT 1 FROM json_each(i.ai_json, '$.qwen') AS q
|
| 34 |
+
WHERE CASE WHEN q.type = 'object' THEN COALESCE(
|
| 35 |
+
json_type(q.value, '$.valid') = 'true'
|
| 36 |
+
AND json_type(q.value, '$.object') = 'text'
|
| 37 |
+
AND json_type(q.value, '$.clarity') = 'integer'
|
| 38 |
+
AND json_extract(q.value, '$.clarity') BETWEEN 0 AND 3, 0)
|
| 39 |
+
ELSE 0 END = 0)
|
| 40 |
+
AND json_type(i.ai_json, '$.evidence.valid') = 'true'
|
| 41 |
+
AND json_extract(i.ai_json, '$.evidence.clarity') >= 2
|
| 42 |
+
AND EXISTS (SELECT 1 FROM json_each(i.ai_json, '$.qwen') AS q
|
| 43 |
+
WHERE CASE WHEN q.type = 'object' THEN
|
| 44 |
+
json_extract(q.value, '$.object') = json_extract(i.ai_json, '$.evidence.object')
|
| 45 |
+
AND json_extract(q.value, '$.clarity') = json_extract(i.ai_json, '$.evidence.clarity')
|
| 46 |
+
ELSE 0 END),
|
| 47 |
+
0) ELSE 0 END;
|
| 48 |
+
|
| 49 |
+
CREATE TRIGGER IF NOT EXISTS accepted_guard_insert
|
| 50 |
+
BEFORE INSERT ON accepted
|
| 51 |
+
WHEN NOT EXISTS (
|
| 52 |
+
SELECT 1 FROM eligible_approval_items AS i
|
| 53 |
+
WHERE i.id = NEW.item_id AND i.label IS NEW.label
|
| 54 |
+
AND i.city_id IS NEW.city_id AND i.design_id IS NEW.design_id
|
| 55 |
+
AND NOT EXISTS (SELECT 1 FROM approval_revocations WHERE item_id = NEW.item_id)
|
| 56 |
+
)
|
| 57 |
+
BEGIN
|
| 58 |
+
SELECT RAISE(ABORT, 'Approval conditions invalid or explicit reapproval required');
|
| 59 |
+
END;
|
| 60 |
+
|
| 61 |
+
CREATE TRIGGER IF NOT EXISTS accepted_guard_update
|
| 62 |
+
BEFORE UPDATE ON accepted
|
| 63 |
+
WHEN NEW.item_id IS NOT OLD.item_id OR NOT EXISTS (
|
| 64 |
+
SELECT 1 FROM eligible_approval_items AS i
|
| 65 |
+
WHERE i.id = NEW.item_id AND i.label IS NEW.label
|
| 66 |
+
AND i.city_id IS NEW.city_id AND i.design_id IS NEW.design_id
|
| 67 |
+
AND NOT EXISTS (SELECT 1 FROM approval_revocations WHERE item_id = NEW.item_id)
|
| 68 |
+
)
|
| 69 |
+
BEGIN
|
| 70 |
+
SELECT RAISE(ABORT, 'Approval conditions invalid or explicit reapproval required');
|
| 71 |
+
END;
|
| 72 |
+
|
| 73 |
+
CREATE TRIGGER IF NOT EXISTS accepted_record_revocation
|
| 74 |
+
AFTER DELETE ON accepted
|
| 75 |
+
BEGIN
|
| 76 |
+
INSERT OR IGNORE INTO approval_revocations
|
| 77 |
+
(item_id, label, revoked_at, reason, previous_accepted_at)
|
| 78 |
+
VALUES (OLD.item_id, OLD.label, strftime('%Y-%m-%dT%H:%M:%fZ','now'),
|
| 79 |
+
CASE WHEN NOT EXISTS (SELECT 1 FROM items WHERE id = OLD.item_id)
|
| 80 |
+
THEN 'source_item_deleted'
|
| 81 |
+
WHEN NOT EXISTS (SELECT 1 FROM eligible_approval_items WHERE id = OLD.item_id)
|
| 82 |
+
THEN 'approval_conditions_invalid'
|
| 83 |
+
ELSE 'accepted_removed' END,
|
| 84 |
+
OLD.accepted_at);
|
| 85 |
+
END;
|
| 86 |
+
|
| 87 |
+
CREATE TRIGGER IF NOT EXISTS items_revoke_invalid_approval
|
| 88 |
+
AFTER UPDATE OF id, kind, status, geometry_ok, seoul_ok, ai_json, label, city_id, design_id,
|
| 89 |
+
graph, record_path, points_json, source_points_json, edge_json, fingerprint ON items
|
| 90 |
+
BEGIN
|
| 91 |
+
INSERT OR IGNORE INTO approval_revocations
|
| 92 |
+
(item_id, label, revoked_at, reason, previous_accepted_at)
|
| 93 |
+
SELECT a.item_id, a.label, strftime('%Y-%m-%dT%H:%M:%fZ','now'),
|
| 94 |
+
CASE WHEN NEW.ai_json IS NOT OLD.ai_json
|
| 95 |
+
THEN 'review_evidence_changed_requires_reapproval'
|
| 96 |
+
WHEN NEW.graph IS NOT OLD.graph OR NEW.record_path IS NOT OLD.record_path
|
| 97 |
+
OR NEW.points_json IS NOT OLD.points_json OR NEW.source_points_json IS NOT OLD.source_points_json
|
| 98 |
+
OR NEW.edge_json IS NOT OLD.edge_json OR NEW.fingerprint IS NOT OLD.fingerprint
|
| 99 |
+
THEN 'route_source_changed_requires_reapproval'
|
| 100 |
+
ELSE 'approval_state_or_identity_changed' END, a.accepted_at
|
| 101 |
+
FROM accepted AS a WHERE a.item_id = OLD.id AND (
|
| 102 |
+
NEW.ai_json IS NOT OLD.ai_json
|
| 103 |
+
OR NEW.graph IS NOT OLD.graph OR NEW.record_path IS NOT OLD.record_path
|
| 104 |
+
OR NEW.points_json IS NOT OLD.points_json OR NEW.source_points_json IS NOT OLD.source_points_json
|
| 105 |
+
OR NEW.edge_json IS NOT OLD.edge_json OR NEW.fingerprint IS NOT OLD.fingerprint
|
| 106 |
+
OR NOT EXISTS (
|
| 107 |
+
SELECT 1 FROM eligible_approval_items AS i
|
| 108 |
+
WHERE i.id = a.item_id AND i.label IS a.label
|
| 109 |
+
AND i.city_id IS a.city_id AND i.design_id IS a.design_id
|
| 110 |
+
)
|
| 111 |
+
);
|
| 112 |
+
DELETE FROM accepted WHERE item_id = OLD.id
|
| 113 |
+
AND EXISTS (SELECT 1 FROM approval_revocations WHERE item_id = OLD.id);
|
| 114 |
+
END;
|
| 115 |
+
|
| 116 |
+
CREATE TRIGGER IF NOT EXISTS revocations_exclude_accepted
|
| 117 |
+
AFTER INSERT ON approval_revocations
|
| 118 |
+
BEGIN
|
| 119 |
+
DELETE FROM accepted WHERE item_id = NEW.item_id;
|
| 120 |
+
END;
|
| 121 |
+
|
| 122 |
+
CREATE TRIGGER IF NOT EXISTS items_revoke_deleted_approval
|
| 123 |
+
AFTER DELETE ON items
|
| 124 |
+
BEGIN
|
| 125 |
+
DELETE FROM accepted WHERE item_id = OLD.id;
|
| 126 |
+
END;
|
| 127 |
+
|
| 128 |
+
INSERT INTO settings(key, value) VALUES ('approval_guards_version', '1')
|
| 129 |
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value;
|
tools/lab_sync/tests/fixtures/approval_policy.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure, CPU-only checks for the existing blind two-model review evidence."""
|
| 2 |
+
import json
|
| 3 |
+
import re
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def category(text):
|
| 7 |
+
"""Keep the reviewer's existing category precedence and vocabulary."""
|
| 8 |
+
value = text.lower().strip()
|
| 9 |
+
if re.search(r'\b(unknown|unrecognizable|nothing|blank|abstract|polygon|scribble|geometric)\b', value):
|
| 10 |
+
return 'unknown'
|
| 11 |
+
for key, pattern in [
|
| 12 |
+
('shorts', r'\bshorts\b|short[ -](trousers|pants)'),
|
| 13 |
+
('sweater', r'long[ -]sleeve|\b(sweater|jacket|coat)\b'),
|
| 14 |
+
('heart', r'\bheart\b|love[ -](symbol|icon)|\bvalentine\b'),
|
| 15 |
+
('tree', r'\b(tree|fir|pine|spruce)\b'),
|
| 16 |
+
('tshirt', r'\bt[ -]?shirt\b|\btee\b|\bshirt\b'),
|
| 17 |
+
('pants', r'\b(pants|trousers?|jeans|slacks)\b'),
|
| 18 |
+
('candle', r'\bcandle\b'),
|
| 19 |
+
('dog', r'\b(dog|puppy|hound|canine|terrier|dachshund|poodle|corgi|beagle|labrador|retriever|bulldog|dalmatian|husky|shiba|schnauzer|chihuahua)\b'),
|
| 20 |
+
('cat', r'\b(cat|kitten|feline)\b'),
|
| 21 |
+
('circle', r'\b(circle|oval)\b'),
|
| 22 |
+
('square', r'\b(square|rectangle)\b'),
|
| 23 |
+
]:
|
| 24 |
+
if re.search(pattern, value):
|
| 25 |
+
return key
|
| 26 |
+
return 'star' if re.search(r'\bstars?\b', value) else 'other'
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def valid_review(encoded, label):
|
| 30 |
+
try:
|
| 31 |
+
review = json.loads(encoded)
|
| 32 |
+
if not isinstance(review, dict):
|
| 33 |
+
return False
|
| 34 |
+
protocol = review.get('protocol')
|
| 35 |
+
if (review.get('status') != 'ai_pass' or review.get('expected_label') != label
|
| 36 |
+
or review.get('blind') is not True or review.get('qwen_performed') is not True
|
| 37 |
+
or not isinstance(protocol, str) or not protocol.strip()
|
| 38 |
+
or not review.get('siglip_support_angles')):
|
| 39 |
+
return False
|
| 40 |
+
views = review.get('siglip_views')
|
| 41 |
+
if not isinstance(views, list) or not any(
|
| 42 |
+
isinstance(v, dict) and v.get('top_label') == label for v in views):
|
| 43 |
+
return False
|
| 44 |
+
responses = review.get('qwen')
|
| 45 |
+
if not isinstance(responses, list) or not responses:
|
| 46 |
+
return False
|
| 47 |
+
if not all(isinstance(q, dict) and q.get('valid') is True
|
| 48 |
+
and isinstance(q.get('object'), str)
|
| 49 |
+
and type(q.get('clarity')) is int and 0 <= q['clarity'] <= 3
|
| 50 |
+
for q in responses):
|
| 51 |
+
return False
|
| 52 |
+
return any(category(q['object']) == label and q['clarity'] >= 2 for q in responses)
|
| 53 |
+
except (ValueError, TypeError, AttributeError):
|
| 54 |
+
return False
|
tools/lab_sync/tests/test_lab_sender_v2_offline.py
ADDED
|
@@ -0,0 +1,617 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Synthetic fixtures, in-memory HTTP peer, and sockets denied throughout."""
|
| 2 |
+
import base64
|
| 3 |
+
from contextlib import redirect_stdout
|
| 4 |
+
import copy
|
| 5 |
+
import hashlib
|
| 6 |
+
import importlib.util
|
| 7 |
+
import io
|
| 8 |
+
import json
|
| 9 |
+
import math
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import shutil
|
| 12 |
+
import socket
|
| 13 |
+
import sqlite3
|
| 14 |
+
import sys
|
| 15 |
+
import urllib.error
|
| 16 |
+
from urllib.parse import urlsplit
|
| 17 |
+
import uuid
|
| 18 |
+
|
| 19 |
+
import pytest
|
| 20 |
+
|
| 21 |
+
WORK = Path(__file__).resolve().parents[1]
|
| 22 |
+
SENDER = WORK / 'sync_lab_gpx.py'
|
| 23 |
+
REPO = WORK.parents[1]
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def load(name, path):
|
| 27 |
+
spec = importlib.util.spec_from_file_location(name, path)
|
| 28 |
+
module = importlib.util.module_from_spec(spec)
|
| 29 |
+
spec.loader.exec_module(module)
|
| 30 |
+
return module
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
sender = load('lab_sender_offline', SENDER)
|
| 34 |
+
schemas = load('receiver_contract_offline', REPO / 'api/ai-service/app/schemas/route_library.py')
|
| 35 |
+
sys.path.insert(0, str(REPO / 'api/ai-service'))
|
| 36 |
+
from app.services import art_route_ingest as ingest
|
| 37 |
+
|
| 38 |
+
SCHEMA = '''
|
| 39 |
+
CREATE TABLE items(id TEXT PRIMARY KEY,kind TEXT,label TEXT,city_id TEXT,city_name TEXT,
|
| 40 |
+
design_id TEXT,status TEXT,geometry_ok INTEGER,seoul_ok INTEGER,ai_json TEXT,priority REAL DEFAULT 0,
|
| 41 |
+
graph TEXT,record_path TEXT,points_json TEXT,source_points_json TEXT,edge_json TEXT,fingerprint TEXT);
|
| 42 |
+
CREATE TABLE accepted(item_id TEXT PRIMARY KEY,label TEXT,city_id TEXT,design_id TEXT,accepted_at TEXT);
|
| 43 |
+
CREATE TABLE settings(key TEXT PRIMARY KEY,value TEXT);
|
| 44 |
+
'''
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@pytest.fixture(autouse=True)
|
| 48 |
+
def deny_io(monkeypatch):
|
| 49 |
+
def forbidden(*args, **kwargs):
|
| 50 |
+
raise AssertionError('Network and real HTTP are forbidden in offline tests')
|
| 51 |
+
monkeypatch.setattr(socket.socket, 'connect', forbidden)
|
| 52 |
+
monkeypatch.setattr(socket.socket, 'connect_ex', forbidden)
|
| 53 |
+
monkeypatch.setattr(socket, 'create_connection', forbidden)
|
| 54 |
+
monkeypatch.setattr(sender.urllib.request, 'urlopen', forbidden)
|
| 55 |
+
monkeypatch.setattr(ingest, 'outline_from_points', lambda points, name: [[0, 0], [1, 0], [0, 1]])
|
| 56 |
+
monkeypatch.setattr(sender.time, 'sleep', lambda seconds: None)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def review(label='heart'):
|
| 60 |
+
q = {'valid': True, 'object': label, 'clarity': 3}
|
| 61 |
+
return {'status': 'ai_pass', 'expected_label': label, 'blind': True,
|
| 62 |
+
'qwen_performed': True, 'protocol': 'synthetic-blind-v2',
|
| 63 |
+
'siglip_support_angles': [0], 'siglip_views': [{'top_label': label, 'angle_cw': 0}],
|
| 64 |
+
'qwen': [q], 'evidence': q}
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class Lab:
|
| 68 |
+
def __init__(self, root):
|
| 69 |
+
self.root = root
|
| 70 |
+
self.state = root / 'state.sqlite3'
|
| 71 |
+
self.url = 'https://synthetic.invalid/api/v1/ai/internal/route-library'
|
| 72 |
+
for name in ['approval_policy.py', 'approval_guards.sql']:
|
| 73 |
+
shutil.copy2(WORK / 'tests/fixtures' / name, root / name)
|
| 74 |
+
with sqlite3.connect(root / 'inventory.sqlite3') as db:
|
| 75 |
+
db.executescript(SCHEMA + (root / 'approval_guards.sql').read_text())
|
| 76 |
+
with sqlite3.connect(self.state) as db:
|
| 77 |
+
db.execute('CREATE TABLE sent(id TEXT PRIMARY KEY,signature TEXT NOT NULL)')
|
| 78 |
+
self.ids = []
|
| 79 |
+
self.design_ids = {}
|
| 80 |
+
self.records = {}
|
| 81 |
+
self.add('a' * 64)
|
| 82 |
+
self.add('b' * 64)
|
| 83 |
+
|
| 84 |
+
def add(self, key, design_status='ai_reject', accept=True):
|
| 85 |
+
label = 'heart'
|
| 86 |
+
design = {'design_id': 'synthetic_heart', 'version': 1, 'name': 'Synthetic heart',
|
| 87 |
+
'outline': {'coordinate_system': 'cartesian_y_up'},
|
| 88 |
+
'points': [[0, 0], [1, 0], [1, 1], [0, 1]]}
|
| 89 |
+
design_digest = hashlib.sha256(sender.canonical(design).encode()).hexdigest()
|
| 90 |
+
design_item = hashlib.sha256(('design:' + label + ':' + design_digest).encode()).hexdigest()
|
| 91 |
+
design_path = self.root / 'designs/heart/synthetic_heart.json'
|
| 92 |
+
design_path.parent.mkdir(parents=True, exist_ok=True)
|
| 93 |
+
design_path.write_text(sender.canonical(design))
|
| 94 |
+
# Every coordinate below is fabricated, never loaded from a real Lab record.
|
| 95 |
+
points = [[12 + .001 * math.sin(i * math.tau / 20), 34 + .001 * math.cos(i * math.tau / 20)] for i in range(20)]
|
| 96 |
+
points.append(points[0])
|
| 97 |
+
gps = [[lon, lat] for lat, lon in points]
|
| 98 |
+
gpx = ('<gpx><trk><trkseg>' + ''.join(f'<trkpt lat="{lat}" lon="{lon}"/>' for lat, lon in points)
|
| 99 |
+
+ '</trkseg></trk></gpx>').encode()
|
| 100 |
+
gpx_path = self.root / 'gpx/heart' / (key + '.gpx')
|
| 101 |
+
gpx_path.parent.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
gpx_path.write_bytes(gpx)
|
| 103 |
+
record = {'metadata': {'design_file': str(design_path), 'design_digest': design_digest, 'graph_sha256': 'c' * 64},
|
| 104 |
+
'route': {'design_id': design['design_id'], 'design_version': 1, 'length_m': 1500.4,
|
| 105 |
+
'gps': gps, 'verification': {'road_only': True, 'graph_connectivity_verified': True,
|
| 106 |
+
'access_checked': True, 'directions_checked': True, 'air_length_m': 0}},
|
| 107 |
+
'assessment': {'success': True, 'route_valid': True}}
|
| 108 |
+
record_path = self.root / (key + '.json')
|
| 109 |
+
record_path.write_text(sender.canonical(record))
|
| 110 |
+
with sqlite3.connect(self.root / 'inventory.sqlite3') as db:
|
| 111 |
+
db.execute('INSERT OR IGNORE INTO items(id,kind,label,status,ai_json) VALUES (?,?,?,?,?)',
|
| 112 |
+
(design_item, 'design', label, design_status,
|
| 113 |
+
json.dumps(review() if design_status == 'ai_pass' else {'status': design_status})))
|
| 114 |
+
db.execute('INSERT INTO items(id,kind,label,city_id,city_name,design_id,status,geometry_ok,seoul_ok,ai_json,record_path) VALUES (?,?,?,?,?,?,?,?,?,?,?)',
|
| 115 |
+
(key, 'route', label, 'synthetic', 'Synthetic region', design['design_id'], 'ai_pass', 1, 1,
|
| 116 |
+
json.dumps(review()), str(record_path)))
|
| 117 |
+
if accept:
|
| 118 |
+
db.execute('INSERT INTO accepted VALUES (?,?,?,?,?)', (key, label, 'synthetic', design['design_id'], '2026-11-10T00:00:00Z'))
|
| 119 |
+
self.ids.append(key)
|
| 120 |
+
self.design_ids[key] = design_item
|
| 121 |
+
self.records[key] = record_path
|
| 122 |
+
|
| 123 |
+
def seed_v1(self, keys):
|
| 124 |
+
with sqlite3.connect(self.state) as db:
|
| 125 |
+
for key in keys:
|
| 126 |
+
db.execute('INSERT INTO sent VALUES (?,?)', (key, 'legacy-signature'))
|
| 127 |
+
|
| 128 |
+
def run(self, **kwargs):
|
| 129 |
+
return sender.sync_v2(self.root, self.url, 'synthetic-auth', self.state, **kwargs)
|
| 130 |
+
|
| 131 |
+
def sql(self, sql, args=()):
|
| 132 |
+
with sqlite3.connect(self.state) as db:
|
| 133 |
+
db.row_factory = sqlite3.Row
|
| 134 |
+
return [dict(r) for r in db.execute(sql, args)]
|
| 135 |
+
|
| 136 |
+
def revoke(self, key):
|
| 137 |
+
with sqlite3.connect(self.root / 'inventory.sqlite3') as db:
|
| 138 |
+
db.execute('DELETE FROM accepted WHERE item_id=?', (key,))
|
| 139 |
+
|
| 140 |
+
def change_record(self, key, operation):
|
| 141 |
+
record = json.loads(self.records[key].read_text())
|
| 142 |
+
operation(record)
|
| 143 |
+
self.records[key].write_text(sender.canonical(record))
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
@pytest.fixture
|
| 147 |
+
def lab(tmp_path):
|
| 148 |
+
return Lab(tmp_path)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
class Peer:
|
| 152 |
+
"""Validates requests using the prepared server models and pure ingest code."""
|
| 153 |
+
def __init__(self):
|
| 154 |
+
self.calls = []
|
| 155 |
+
self.generation = 0
|
| 156 |
+
self.sessions = {}
|
| 157 |
+
self.current = None
|
| 158 |
+
self.stored = {}
|
| 159 |
+
self.template_ids = {}
|
| 160 |
+
|
| 161 |
+
def __call__(self, url, token, method, payload):
|
| 162 |
+
self.calls.append((method, urlsplit(url).path, copy.deepcopy(payload)))
|
| 163 |
+
path = urlsplit(url).path
|
| 164 |
+
if path.endswith('/sync-state'):
|
| 165 |
+
assert method == 'GET' and payload is None
|
| 166 |
+
return {'contract_version': 2, 'generation': self.generation, 'catalog_revision': 0, 'v2_required': bool(self.generation)}
|
| 167 |
+
if path.endswith('/sync-sessions'):
|
| 168 |
+
data = schemas.SessionStart.model_validate(payload)
|
| 169 |
+
assert data.manifest_sha256 == ingest.sha(data.manifest)
|
| 170 |
+
key = str(data.client_request_id)
|
| 171 |
+
if key in self.sessions:
|
| 172 |
+
session = self.sessions[key]
|
| 173 |
+
if session['session_id'] != self.current:
|
| 174 |
+
raise urllib.error.HTTPError(url, 409, 'stale', None, None)
|
| 175 |
+
else:
|
| 176 |
+
if data.expected_generation != self.generation:
|
| 177 |
+
raise urllib.error.HTTPError(url, 409, 'stale', None, None)
|
| 178 |
+
self.generation += 1
|
| 179 |
+
session = {'contract_version': 2, 'session_id': str(uuid.uuid4()), 'generation': self.generation,
|
| 180 |
+
'mode': data.mode, 'session_status': 'open', 'manifest_sha256': data.manifest_sha256}
|
| 181 |
+
self.sessions[key] = session
|
| 182 |
+
self.current = session['session_id']
|
| 183 |
+
return dict(session)
|
| 184 |
+
session = next(s for s in self.sessions.values() if s['session_id'] == self.current)
|
| 185 |
+
if '/sync-sessions/' in path:
|
| 186 |
+
if path.endswith('/complete'):
|
| 187 |
+
assert method == 'POST' and session['mode'] == 'partial'
|
| 188 |
+
assert urlsplit(url).query == 'generation=' + str(session['generation'])
|
| 189 |
+
session['session_status'] = 'completed'
|
| 190 |
+
return {**session, 'deactivated_count': 0}
|
| 191 |
+
assert method == 'GET' and path.endswith(self.current)
|
| 192 |
+
return dict(session)
|
| 193 |
+
if '/routes/' in path:
|
| 194 |
+
assert method == 'PUT'
|
| 195 |
+
key = path.rsplit('/', 1)[-1]
|
| 196 |
+
data = schemas.RouteImportV2.model_validate(payload)
|
| 197 |
+
assert str(data.session_id) == self.current and data.generation == self.generation
|
| 198 |
+
assert sender.content_sha(payload) == ingest.content_sha(data)
|
| 199 |
+
prepared = ingest.prepare_route(key, data)
|
| 200 |
+
# The response ID is an arbitrary server ID, never echoed from an input ID.
|
| 201 |
+
template_key = data.template.design_digest
|
| 202 |
+
self.template_ids.setdefault(template_key, str(uuid.uuid4()))
|
| 203 |
+
old = self.stored.get(key)
|
| 204 |
+
candidate_id = old['art_route_candidate_id'] if old else str(uuid.uuid4())
|
| 205 |
+
receipt = {'contract_version': 2, 'status': 'stored', 'result': 'unchanged' if old else 'created',
|
| 206 |
+
'source_id': key, 'sha256': data.sha256, 'payload_sha256': data.payload_sha256,
|
| 207 |
+
'art_template_id': self.template_ids[template_key], 'art_route_candidate_id': candidate_id,
|
| 208 |
+
'design_ai_status': data.template.design_ai_status, 'route_validation_status': 'passed'}
|
| 209 |
+
assert prepared['coordinates'] != data.template.snapshot['points']
|
| 210 |
+
self.stored[key] = receipt
|
| 211 |
+
return dict(receipt)
|
| 212 |
+
assert path.endswith('/approval-snapshot') and method == 'POST'
|
| 213 |
+
data = schemas.ApprovalSnapshotV2.model_validate(payload)
|
| 214 |
+
assert data.count == len(data.ids) and data.manifest_sha256 == session['manifest_sha256']
|
| 215 |
+
assert data.ids_sha256 == hashlib.sha256('\n'.join(data.ids).encode()).hexdigest()
|
| 216 |
+
session['session_status'] = 'completed'
|
| 217 |
+
return {'contract_version': 2, 'session_id': self.current, 'generation': self.generation,
|
| 218 |
+
'active_count': data.count, 'deactivated_count': 0}
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
@pytest.fixture
|
| 222 |
+
def peer(monkeypatch):
|
| 223 |
+
peer = Peer()
|
| 224 |
+
monkeypatch.setattr(sender, 'send', peer)
|
| 225 |
+
return peer
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def puts(peer):
|
| 229 |
+
return [c for c in peer.calls if c[0] == 'PUT']
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def test_v1_history_is_preserved_but_every_v2_receipt_is_required(lab, peer):
|
| 233 |
+
lab.seed_v1(lab.ids)
|
| 234 |
+
before = lab.sql('SELECT * FROM sent ORDER BY id')
|
| 235 |
+
result = lab.run()
|
| 236 |
+
assert result['uploaded'] == 2 and result['snapshot_sent']
|
| 237 |
+
assert lab.sql('SELECT * FROM sent ORDER BY id') == before
|
| 238 |
+
receipts = lab.sql('SELECT * FROM http_v2_sent')
|
| 239 |
+
assert len(receipts) == 2
|
| 240 |
+
for receipt in receipts:
|
| 241 |
+
remote = peer.stored[receipt['id']]
|
| 242 |
+
assert receipt['template_id'] == remote['art_template_id']
|
| 243 |
+
assert receipt['candidate_id'] == remote['art_route_candidate_id']
|
| 244 |
+
assert len([c for c in peer.calls if c[0] == 'GET' and '/sync-sessions/' in c[1]]) == 2
|
| 245 |
+
packet = puts(peer)[0][2]
|
| 246 |
+
assert packet['template']['design_ai_status'] == 'ai_reject'
|
| 247 |
+
assert packet['route_ai_evidence']['status'] == 'ai_pass'
|
| 248 |
+
assert 'coordinates' not in packet and 'gps' not in packet
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
@pytest.mark.parametrize('status', ['ai_pass', 'ai_review', 'unknown'])
|
| 252 |
+
def test_independent_design_states(lab, peer, status):
|
| 253 |
+
with sqlite3.connect(lab.root / 'inventory.sqlite3') as db:
|
| 254 |
+
db.execute('UPDATE items SET status=?,ai_json=? WHERE kind=?',
|
| 255 |
+
(status, json.dumps(review() if status == 'ai_pass' else {'status': status}), 'design'))
|
| 256 |
+
lab.run(probe_id=lab.ids[0])
|
| 257 |
+
assert puts(peer)[0][2]['template']['design_ai_status'] == status
|
| 258 |
+
assert not any(c[1].endswith('/approval-snapshot') for c in peer.calls)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def test_readonly_preflight_counts_v1_as_pending_without_creating_schema(lab, monkeypatch):
|
| 262 |
+
lab.seed_v1(lab.ids)
|
| 263 |
+
before = lab.state.read_bytes(), lab.state.stat().st_mtime_ns
|
| 264 |
+
monkeypatch.setattr(sender, 'send', lambda *a: pytest.fail('HTTP forbidden'))
|
| 265 |
+
result = sender.check_only_v2(lab.root, lab.state, lab.url)
|
| 266 |
+
assert result['v1_history'] == 2 and result['v2_confirmed'] == 0 and result['v2_pending'] == 2
|
| 267 |
+
assert result['http_requests'] == result['credential_reads'] == result['state_writes'] == 0
|
| 268 |
+
assert (lab.state.read_bytes(), lab.state.stat().st_mtime_ns) == before
|
| 269 |
+
assert [r['name'] for r in lab.sql("SELECT name FROM sqlite_master WHERE type='table'")] == ['sent']
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def test_cli_check_only_reads_no_credential_lock_or_status(lab, monkeypatch):
|
| 273 |
+
before = lab.state.read_bytes()
|
| 274 |
+
monkeypatch.setattr(sender, 'DEFAULT_STATE', lab.state)
|
| 275 |
+
monkeypatch.setattr(sender, 'single_instance', lambda: pytest.fail('No lock mutation in preflight'))
|
| 276 |
+
monkeypatch.setattr(sender, 'check_health', lambda *a: pytest.fail('No health request in preflight'))
|
| 277 |
+
monkeypatch.setattr(sys, 'argv', ['sync', '--root', str(lab.root), '--url', lab.url,
|
| 278 |
+
'--token-file', str(lab.root / 'DOES_NOT_EXIST'), '--state', str(lab.state), '--contract-version', '2', '--check-only'])
|
| 279 |
+
output = io.StringIO()
|
| 280 |
+
with redirect_stdout(output):
|
| 281 |
+
sender.main()
|
| 282 |
+
assert json.loads(output.getvalue())['v2_pending'] == 2
|
| 283 |
+
assert lab.state.read_bytes() == before
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def test_cap_retains_v1_history_instead_of_first_slice(lab, peer):
|
| 287 |
+
lab.seed_v1([lab.ids[1]])
|
| 288 |
+
result = lab.run(max_routes=1)
|
| 289 |
+
assert result['selected'] == result['uploaded'] == result['sent_total'] == 1
|
| 290 |
+
assert puts(peer)[0][1].endswith(lab.ids[1])
|
| 291 |
+
assert not result['snapshot_sent']
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
@pytest.mark.parametrize('case', ['overflow', 'revoked', 'missing_state', 'missing_guards', 'empty', 'unapproved_probe'])
|
| 295 |
+
def test_early_failures_make_no_http_calls(lab, peer, case):
|
| 296 |
+
kwargs = {}
|
| 297 |
+
if case == 'overflow':
|
| 298 |
+
lab.seed_v1(lab.ids); kwargs['max_routes'] = 1
|
| 299 |
+
elif case == 'revoked':
|
| 300 |
+
lab.seed_v1([lab.ids[0]]); lab.revoke(lab.ids[0]); kwargs['max_routes'] = 2
|
| 301 |
+
elif case == 'missing_state':
|
| 302 |
+
lab.state.unlink()
|
| 303 |
+
elif case == 'missing_guards':
|
| 304 |
+
with sqlite3.connect(lab.root / 'inventory.sqlite3') as db:
|
| 305 |
+
db.execute('DELETE FROM settings')
|
| 306 |
+
elif case == 'empty':
|
| 307 |
+
for key in lab.ids: lab.revoke(key)
|
| 308 |
+
else:
|
| 309 |
+
kwargs['probe_id'] = 'f' * 64
|
| 310 |
+
with pytest.raises((RuntimeError, sqlite3.Error)):
|
| 311 |
+
lab.run(**kwargs)
|
| 312 |
+
assert peer.calls == []
|
| 313 |
+
if case == 'missing_state': assert not lab.state.exists()
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
@pytest.mark.parametrize('case', ['design_digest', 'road_evidence', 'graph_hash', 'gps_mismatch', 'source_missing',
|
| 317 |
+
'design_review', 'design_status', 'assessment_text'])
|
| 318 |
+
def test_invalid_source_material_is_rejected_before_http(lab, peer, case):
|
| 319 |
+
key = lab.ids[0]
|
| 320 |
+
if case == 'design_digest':
|
| 321 |
+
lab.change_record(key, lambda r: r['metadata'].update(design_digest='d' * 64))
|
| 322 |
+
elif case == 'road_evidence':
|
| 323 |
+
lab.change_record(key, lambda r: r['route']['verification'].update(access_checked=False))
|
| 324 |
+
elif case == 'graph_hash':
|
| 325 |
+
lab.change_record(key, lambda r: r['metadata'].update(graph_sha256='wrong'))
|
| 326 |
+
elif case == 'gps_mismatch':
|
| 327 |
+
lab.change_record(key, lambda r: r['route']['gps'][0].__setitem__(0, 50))
|
| 328 |
+
elif case == 'source_missing':
|
| 329 |
+
lab.records[key].unlink()
|
| 330 |
+
elif case == 'assessment_text':
|
| 331 |
+
lab.change_record(key, lambda r: r['assessment'].update(success='false'))
|
| 332 |
+
else:
|
| 333 |
+
with sqlite3.connect(lab.root / 'inventory.sqlite3') as db:
|
| 334 |
+
db.execute('UPDATE items SET status=?,ai_json=? WHERE kind=?',
|
| 335 |
+
('ai_pass' if case == 'design_review' else 'ai_reject', json.dumps({}), 'design'))
|
| 336 |
+
if case == 'design_status':
|
| 337 |
+
db.execute('UPDATE items SET ai_json=? WHERE kind=?', (json.dumps({'status': 'ai_pass'}), 'design'))
|
| 338 |
+
with pytest.raises((RuntimeError, OSError)):
|
| 339 |
+
lab.run()
|
| 340 |
+
assert peer.calls == []
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def test_semantic_route_policy_cannot_be_bypassed_by_structural_guard(lab, peer):
|
| 344 |
+
key = 'c' * 64
|
| 345 |
+
lab.add(key, accept=False)
|
| 346 |
+
wrong = review(); wrong['qwen'][0]['object'] = 'cat'
|
| 347 |
+
with sqlite3.connect(lab.root / 'inventory.sqlite3') as db:
|
| 348 |
+
db.execute('UPDATE items SET ai_json=? WHERE id=?', (json.dumps(wrong), key))
|
| 349 |
+
db.execute('INSERT INTO accepted VALUES (?,?,?,?,?)', (key, 'heart', 'synthetic', 'synthetic_heart', '2026-11-10T00:00:00Z'))
|
| 350 |
+
with pytest.raises(RuntimeError): lab.run()
|
| 351 |
+
assert peer.calls == []
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
@pytest.mark.parametrize('field,value', [
|
| 355 |
+
('contract_version', 1), ('art_template_id', None), ('art_route_candidate_id', 'invalid'),
|
| 356 |
+
('art_template_id', str(uuid.UUID(int=0))), ('source_id', 'f' * 64), ('sha256', 'f' * 64),
|
| 357 |
+
('payload_sha256', 'f' * 64), ('design_ai_status', 'ai_pass'), ('route_validation_status', 'pending'),
|
| 358 |
+
('status', 'ok'), ('result', 'queued'),
|
| 359 |
+
])
|
| 360 |
+
def test_invalid_receipt_never_marks_v2_sent(lab, peer, monkeypatch, field, value):
|
| 361 |
+
def response(url, token, method, payload):
|
| 362 |
+
receipt = peer(url, token, method, payload)
|
| 363 |
+
if method == 'PUT': receipt[field] = value
|
| 364 |
+
return receipt
|
| 365 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 366 |
+
with pytest.raises(sender.ReceiverUnhealthy): lab.run()
|
| 367 |
+
assert lab.sql('SELECT * FROM http_v2_sent') == []
|
| 368 |
+
assert not any(c[1].endswith('/approval-snapshot') for c in peer.calls)
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def test_old_server_success_is_not_art_success(lab, peer, monkeypatch):
|
| 372 |
+
def response(url, token, method, payload):
|
| 373 |
+
receipt = peer(url, token, method, payload)
|
| 374 |
+
if method == 'PUT': return {k: receipt[k] for k in ['status', 'source_id', 'sha256']}
|
| 375 |
+
return receipt
|
| 376 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 377 |
+
with pytest.raises(sender.ReceiverUnhealthy): lab.run()
|
| 378 |
+
assert lab.sql('SELECT * FROM http_v2_sent') == []
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
@pytest.mark.parametrize('field,value', [('generation', 99), ('contract_version', 1), ('manifest_sha256', 'f' * 64),
|
| 382 |
+
('session_status', 'superseded'), ('mode', 'partial'), ('session_id', 'invalid')])
|
| 383 |
+
def test_inconsistent_session_is_rejected_before_put(lab, peer, monkeypatch, field, value):
|
| 384 |
+
def response(url, token, method, payload):
|
| 385 |
+
receipt = peer(url, token, method, payload)
|
| 386 |
+
if method == 'POST' and url.endswith('/sync-sessions'): receipt[field] = value
|
| 387 |
+
return receipt
|
| 388 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 389 |
+
with pytest.raises(sender.ReceiverUnhealthy): lab.run()
|
| 390 |
+
assert puts(peer) == []
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def test_session_status_is_checked_before_put(lab, peer, monkeypatch):
|
| 394 |
+
def response(url, token, method, payload):
|
| 395 |
+
receipt = peer(url, token, method, payload)
|
| 396 |
+
if method == 'GET' and '/sync-sessions/' in url: receipt['generation'] += 1
|
| 397 |
+
return receipt
|
| 398 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 399 |
+
with pytest.raises(sender.ReceiverUnhealthy): lab.run()
|
| 400 |
+
assert puts(peer) == []
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
def test_lost_session_response_reuses_request_id_across_restart(lab, peer, monkeypatch):
|
| 404 |
+
def response(url, token, method, payload):
|
| 405 |
+
receipt = peer(url, token, method, payload)
|
| 406 |
+
if method == 'POST' and url.endswith('/sync-sessions'): raise TimeoutError()
|
| 407 |
+
return receipt
|
| 408 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 409 |
+
with pytest.raises(TimeoutError): lab.run()
|
| 410 |
+
assert peer.generation == 1
|
| 411 |
+
request_id = lab.sql('SELECT client_request_id FROM http_v2_runs')[0]['client_request_id']
|
| 412 |
+
monkeypatch.setattr(sender, 'send', peer)
|
| 413 |
+
lab.run()
|
| 414 |
+
assert peer.generation == 1
|
| 415 |
+
assert {c[2]['client_request_id'] for c in peer.calls if c[0] == 'POST' and c[1].endswith('/sync-sessions')} == {request_id}
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def test_lost_put_response_is_retried_with_identical_payload(lab, peer, monkeypatch):
|
| 419 |
+
lost = False
|
| 420 |
+
def response(url, token, method, payload):
|
| 421 |
+
nonlocal lost
|
| 422 |
+
receipt = peer(url, token, method, payload)
|
| 423 |
+
if method == 'PUT' and not lost:
|
| 424 |
+
lost = True
|
| 425 |
+
raise TimeoutError()
|
| 426 |
+
return receipt
|
| 427 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 428 |
+
lab.run(probe_id=lab.ids[0])
|
| 429 |
+
assert len(peer.stored) == 1 and len(puts(peer)) == 2
|
| 430 |
+
assert puts(peer)[0][2] == puts(peer)[1][2]
|
| 431 |
+
assert len(lab.sql('SELECT * FROM http_v2_sent')) == 1
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def test_partial_failure_resumes_verified_receipts(lab, peer, monkeypatch):
|
| 435 |
+
def response(url, token, method, payload):
|
| 436 |
+
if method == 'PUT' and url.endswith(lab.ids[1]): raise TimeoutError()
|
| 437 |
+
return peer(url, token, method, payload)
|
| 438 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 439 |
+
with pytest.raises(TimeoutError): lab.run(max_routes=2)
|
| 440 |
+
assert len(lab.sql('SELECT * FROM http_v2_sent')) == 1
|
| 441 |
+
monkeypatch.setattr(sender, 'send', peer)
|
| 442 |
+
result = lab.run(max_routes=2)
|
| 443 |
+
assert result['uploaded'] == 1 and peer.generation == 1
|
| 444 |
+
assert [c[1].rsplit('/', 1)[-1] for c in puts(peer)] == lab.ids
|
| 445 |
+
assert not result['snapshot_sent']
|
| 446 |
+
|
| 447 |
+
|
| 448 |
+
def test_409_is_not_retried_or_replaced_with_new_session(lab, peer, monkeypatch):
|
| 449 |
+
attempts = []
|
| 450 |
+
def response(url, token, method, payload):
|
| 451 |
+
if method == 'PUT':
|
| 452 |
+
attempts.append(method)
|
| 453 |
+
raise urllib.error.HTTPError(url, 409, 'conflict', None, None)
|
| 454 |
+
return peer(url, token, method, payload)
|
| 455 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 456 |
+
with pytest.raises(urllib.error.HTTPError): lab.run()
|
| 457 |
+
assert attempts == ['PUT'] and peer.generation == 1
|
| 458 |
+
assert lab.sql('SELECT * FROM http_v2_sent') == []
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
@pytest.mark.parametrize('change', ['revocation', 'gpx', 'design_evidence', 'finalized'])
|
| 462 |
+
def test_change_before_retry_aborts_without_resending(lab, peer, monkeypatch, change):
|
| 463 |
+
attempts = []
|
| 464 |
+
def response(url, token, method, payload):
|
| 465 |
+
if method == 'PUT':
|
| 466 |
+
attempts.append(method)
|
| 467 |
+
if change == 'revocation': lab.revoke(lab.ids[0])
|
| 468 |
+
elif change == 'gpx': (lab.root / 'gpx/heart' / (lab.ids[0] + '.gpx')).write_bytes(b'changed')
|
| 469 |
+
elif change == 'finalized': (lab.root / 'completion.json').touch()
|
| 470 |
+
else:
|
| 471 |
+
with sqlite3.connect(lab.root / 'inventory.sqlite3') as db:
|
| 472 |
+
db.execute('UPDATE items SET ai_json=? WHERE kind=?', (json.dumps({'status': 'ai_reject', 'extra': True}), 'design'))
|
| 473 |
+
raise TimeoutError()
|
| 474 |
+
return peer(url, token, method, payload)
|
| 475 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 476 |
+
with pytest.raises(sender.ApprovalChanged): lab.run()
|
| 477 |
+
assert attempts == ['PUT']
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
def test_snapshot_failure_keeps_receipts_and_resumes_same_session(lab, peer, monkeypatch):
|
| 481 |
+
def response(url, token, method, payload):
|
| 482 |
+
if url.endswith('/approval-snapshot'): raise TimeoutError()
|
| 483 |
+
return peer(url, token, method, payload)
|
| 484 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 485 |
+
with pytest.raises(TimeoutError): lab.run()
|
| 486 |
+
assert len(lab.sql('SELECT * FROM http_v2_sent')) == 2
|
| 487 |
+
monkeypatch.setattr(sender, 'send', peer)
|
| 488 |
+
assert lab.run()['uploaded'] == 0
|
| 489 |
+
assert peer.generation == 1
|
| 490 |
+
|
| 491 |
+
|
| 492 |
+
def test_new_session_rechecks_receipts_instead_of_skipping(lab, peer):
|
| 493 |
+
lab.run()
|
| 494 |
+
assert lab.run()['uploaded'] == 2 and peer.generation == 2
|
| 495 |
+
assert len(peer.stored) == 2
|
| 496 |
+
|
| 497 |
+
|
| 498 |
+
def test_explicit_replacement_does_not_rediscover_superseded_checkpoint(lab, peer, monkeypatch):
|
| 499 |
+
def response(url, token, method, payload):
|
| 500 |
+
if method == 'PUT': raise TimeoutError()
|
| 501 |
+
return peer(url, token, method, payload)
|
| 502 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 503 |
+
with pytest.raises(TimeoutError): lab.run(probe_id=lab.ids[0])
|
| 504 |
+
monkeypatch.setattr(sender, 'send', peer)
|
| 505 |
+
lab.run(probe_id=lab.ids[0], new_session=True)
|
| 506 |
+
assert [r['completed'] for r in lab.sql('SELECT completed FROM http_v2_runs ORDER BY rowid')] == [2, 1]
|
| 507 |
+
lab.run(probe_id=lab.ids[0])
|
| 508 |
+
assert peer.generation == 3
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def test_stale_session_status_409_retains_checkpoint_without_sending(lab, peer, monkeypatch):
|
| 512 |
+
attempts = []
|
| 513 |
+
def response(url, token, method, payload):
|
| 514 |
+
if method == 'GET' and '/sync-sessions/' in url:
|
| 515 |
+
attempts.append(method)
|
| 516 |
+
raise urllib.error.HTTPError(url, 409, 'stale', None, None)
|
| 517 |
+
return peer(url, token, method, payload)
|
| 518 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 519 |
+
with pytest.raises(urllib.error.HTTPError): lab.run()
|
| 520 |
+
assert puts(peer) == [] and attempts == ['GET']
|
| 521 |
+
assert lab.sql('SELECT completed FROM http_v2_runs')[0]['completed'] == 0
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def test_retry_wait_keeps_existing_lock_held(lab, peer, monkeypatch):
|
| 525 |
+
lock = lab.root / 'transfer.lock'
|
| 526 |
+
checked = []
|
| 527 |
+
def sleep(seconds):
|
| 528 |
+
with pytest.raises(sender.AlreadyRunning):
|
| 529 |
+
with sender.single_instance(lock): pass
|
| 530 |
+
checked.append(seconds)
|
| 531 |
+
def response(url, token, method, payload):
|
| 532 |
+
raise TimeoutError()
|
| 533 |
+
monkeypatch.setattr(sender.time, 'sleep', sleep)
|
| 534 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 535 |
+
with sender.single_instance(lock), pytest.raises(TimeoutError): lab.run()
|
| 536 |
+
assert checked == [1, 2]
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def test_other_endpoint_never_reuses_a_v2_receipt(lab, peer):
|
| 540 |
+
lab.run(probe_id=lab.ids[0])
|
| 541 |
+
lab.url = 'https://another-synthetic.invalid/api/v1/ai/internal/route-library'
|
| 542 |
+
result = sender.check_only_v2(lab.root, lab.state, lab.url, probe_id=lab.ids[0])
|
| 543 |
+
assert result['v2_confirmed'] == 0 and result['v2_pending'] == 1
|
| 544 |
+
lab.run(probe_id=lab.ids[0], allow_updates=True)
|
| 545 |
+
assert 'previous_payload_sha256' not in puts(peer)[-1][2]
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
def test_missing_endpoint_on_old_patch_receipts_requires_fresh_ack(lab, peer):
|
| 549 |
+
with sqlite3.connect(lab.state) as db:
|
| 550 |
+
db.execute('CREATE TABLE http_v2_sent(id TEXT PRIMARY KEY,signature TEXT NOT NULL,session_id TEXT NOT NULL,template_id TEXT NOT NULL,candidate_id TEXT NOT NULL)')
|
| 551 |
+
db.execute('INSERT INTO http_v2_sent VALUES (?,?,?,?,?)',
|
| 552 |
+
(lab.ids[0], 'legacy', str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())))
|
| 553 |
+
result = sender.check_only_v2(lab.root, lab.state, lab.url)
|
| 554 |
+
assert result['v2_confirmed'] == 0
|
| 555 |
+
assert lab.run(probe_id=lab.ids[0])['uploaded'] == 1
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def test_successful_local_receipts_remain_readonly_in_preflight(lab, peer):
|
| 559 |
+
lab.run()
|
| 560 |
+
before = lab.state.read_bytes(), lab.state.stat().st_mtime_ns
|
| 561 |
+
result = sender.check_only_v2(lab.root, lab.state, lab.url)
|
| 562 |
+
assert result['v2_confirmed'] == 2 and result['v2_pending'] == 0
|
| 563 |
+
assert (lab.state.read_bytes(), lab.state.stat().st_mtime_ns) == before
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
def test_updates_and_reapproval_are_explicit_and_use_verified_hash(lab, peer):
|
| 567 |
+
lab.run(probe_id=lab.ids[0])
|
| 568 |
+
old = lab.sql('SELECT signature FROM http_v2_sent')[0]['signature']
|
| 569 |
+
lab.change_record(lab.ids[0], lambda r: r['route'].update(length_m=1800))
|
| 570 |
+
lab.run(probe_id=lab.ids[0], allow_updates=True, allow_reapproval=True)
|
| 571 |
+
packet = puts(peer)[-1][2]
|
| 572 |
+
assert packet['previous_payload_sha256'] == old and packet['reapprove'] is True
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
def test_approval_changed_during_last_put_withholds_snapshot(lab, peer, monkeypatch):
|
| 576 |
+
def response(url, token, method, payload):
|
| 577 |
+
receipt = peer(url, token, method, payload)
|
| 578 |
+
if method == 'PUT' and url.endswith(lab.ids[-1]): lab.revoke(lab.ids[0])
|
| 579 |
+
return receipt
|
| 580 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 581 |
+
with pytest.raises(sender.ApprovalChanged): lab.run()
|
| 582 |
+
assert not any(c[1].endswith('/approval-snapshot') for c in peer.calls)
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
def test_same_inode_lock_still_excludes_a_second_sender(lab):
|
| 586 |
+
lock = lab.root / 'transfer.lock'
|
| 587 |
+
with sender.single_instance(lock):
|
| 588 |
+
inode = lock.stat().st_ino
|
| 589 |
+
with pytest.raises(sender.AlreadyRunning):
|
| 590 |
+
with sender.single_instance(lock): pass
|
| 591 |
+
assert lock.stat().st_ino == inode
|
| 592 |
+
with sender.single_instance(lock): assert lock.stat().st_ino == inode
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def test_error_status_suppresses_payload_and_halts(lab, peer, monkeypatch):
|
| 596 |
+
token_path = lab.root / 'fixture-auth'
|
| 597 |
+
token_path.write_text('synthetic-auth')
|
| 598 |
+
monkeypatch.setattr(sender, 'DEFAULT_STATE', lab.state)
|
| 599 |
+
monkeypatch.setattr(sender, 'STATUS_PATH', lab.root / 'status.json')
|
| 600 |
+
monkeypatch.setattr(sender, 'HALT_PATH', lab.root / 'HALTED.json')
|
| 601 |
+
original_lock = sender.single_instance
|
| 602 |
+
monkeypatch.setattr(sender, 'single_instance', lambda: original_lock(lab.root / 'transfer.lock'))
|
| 603 |
+
monkeypatch.setattr(sender, 'check_health', lambda *a: None)
|
| 604 |
+
monkeypatch.setattr(sender.signal, 'signal', lambda *a: None)
|
| 605 |
+
def response(url, token, method, payload):
|
| 606 |
+
raise urllib.error.HTTPError(url, 409, 'SENSITIVE_FIXTURE_BODY', None, io.BytesIO(b'SENSITIVE_FIXTURE_BODY'))
|
| 607 |
+
monkeypatch.setattr(sender, 'send', response)
|
| 608 |
+
monkeypatch.setattr(sys, 'argv', ['sync', '--root', str(lab.root), '--url', lab.url,
|
| 609 |
+
'--token-file', str(token_path), '--state', str(lab.state), '--contract-version', '2', '--probe-id', lab.ids[0]])
|
| 610 |
+
output = io.StringIO()
|
| 611 |
+
with redirect_stdout(output), pytest.raises(SystemExit) as result:
|
| 612 |
+
sender.main()
|
| 613 |
+
assert result.value.code == 1
|
| 614 |
+
text = output.getvalue() + (lab.root / 'status.json').read_text() + (lab.root / 'HALTED.json').read_text()
|
| 615 |
+
assert 'synthetic-auth' not in text and 'SENSITIVE_FIXTURE_BODY' not in text
|
| 616 |
+
assert json.loads((lab.root / 'HALTED.json').read_text())['http_status'] == 409
|
| 617 |
+
with pytest.raises(SystemExit): sender.main()
|
tools/sync_lab_gpx.py
CHANGED
|
@@ -10,18 +10,34 @@ import base64
|
|
| 10 |
import hashlib
|
| 11 |
import json
|
| 12 |
import os
|
|
|
|
|
|
|
| 13 |
import sqlite3
|
| 14 |
import time
|
|
|
|
|
|
|
| 15 |
import urllib.error
|
| 16 |
import urllib.request
|
| 17 |
from pathlib import Path
|
| 18 |
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
def approved_rows(root: Path) -> list[sqlite3.Row]:
|
| 21 |
database = (root / "inventory.sqlite3").resolve()
|
| 22 |
connection = sqlite3.connect(f"file:{database.as_posix()}?mode=ro", uri=True)
|
| 23 |
connection.row_factory = sqlite3.Row
|
| 24 |
try:
|
|
|
|
| 25 |
connection.execute("BEGIN")
|
| 26 |
rows = connection.execute("""
|
| 27 |
SELECT i.id, i.label, i.city_id, i.city_name, i.ai_json,
|
|
@@ -35,15 +51,249 @@ def approved_rows(root: Path) -> list[sqlite3.Row]:
|
|
| 35 |
connection.close()
|
| 36 |
|
| 37 |
|
| 38 |
-
def send(url: str, token: str, method: str, payload: dict) -> dict:
|
| 39 |
request = urllib.request.Request(
|
| 40 |
-
url, json.dumps(payload, separators=(",", ":")).encode(),
|
| 41 |
{"Authorization": "Bearer " + token, "Content-Type": "application/json"}, method=method,
|
| 42 |
)
|
| 43 |
with urllib.request.urlopen(request, timeout=30) as response:
|
| 44 |
return json.load(response)
|
| 45 |
|
| 46 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
def sync(root: Path, destination: str, token: str, state_path: Path,
|
| 48 |
probe_id: str | None = None) -> dict:
|
| 49 |
rows = approved_rows(root)
|
|
@@ -122,7 +372,19 @@ def main() -> None:
|
|
| 122 |
parser.add_argument("--state", type=Path, default=Path.home() / ".local/share/demoservice-sync/state.sqlite3")
|
| 123 |
parser.add_argument("--interval-seconds", type=int, default=0, help="Repeat after this delay; 0 runs once")
|
| 124 |
parser.add_argument("--probe-id", help="Send one approved GPX without an approval snapshot")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
args = parser.parse_args()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
token = args.token_file.read_text(encoding="utf-8").strip()
|
| 127 |
if not token:
|
| 128 |
raise SystemExit("Empty receiver token")
|
|
@@ -130,16 +392,38 @@ def main() -> None:
|
|
| 130 |
parser.error("--interval-seconds must be nonnegative")
|
| 131 |
if args.probe_id and args.interval_seconds:
|
| 132 |
parser.error("--probe-id cannot be used with --interval-seconds")
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
try:
|
| 135 |
-
|
| 136 |
-
except
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
if not args.interval_seconds:
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
time.sleep(args.interval_seconds)
|
| 143 |
|
| 144 |
|
| 145 |
if __name__ == "__main__":
|
|
|
|
| 10 |
import hashlib
|
| 11 |
import json
|
| 12 |
import os
|
| 13 |
+
import math
|
| 14 |
+
import re
|
| 15 |
import sqlite3
|
| 16 |
import time
|
| 17 |
+
import uuid
|
| 18 |
+
import xml.etree.ElementTree as ET
|
| 19 |
import urllib.error
|
| 20 |
import urllib.request
|
| 21 |
from pathlib import Path
|
| 22 |
|
| 23 |
|
| 24 |
+
def canonical(value):
|
| 25 |
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def content_sha(payload):
|
| 29 |
+
data = dict(payload)
|
| 30 |
+
for key in ("session_id", "generation", "payload_sha256", "gpx_base64", "previous_payload_sha256", "reapprove"):
|
| 31 |
+
data.pop(key, None)
|
| 32 |
+
return hashlib.sha256(canonical(data).encode()).hexdigest()
|
| 33 |
+
|
| 34 |
+
|
| 35 |
def approved_rows(root: Path) -> list[sqlite3.Row]:
|
| 36 |
database = (root / "inventory.sqlite3").resolve()
|
| 37 |
connection = sqlite3.connect(f"file:{database.as_posix()}?mode=ro", uri=True)
|
| 38 |
connection.row_factory = sqlite3.Row
|
| 39 |
try:
|
| 40 |
+
connection.execute("PRAGMA query_only=ON")
|
| 41 |
connection.execute("BEGIN")
|
| 42 |
rows = connection.execute("""
|
| 43 |
SELECT i.id, i.label, i.city_id, i.city_name, i.ai_json,
|
|
|
|
| 51 |
connection.close()
|
| 52 |
|
| 53 |
|
| 54 |
+
def send(url: str, token: str, method: str, payload: dict | None) -> dict:
|
| 55 |
request = urllib.request.Request(
|
| 56 |
+
url, json.dumps(payload, separators=(",", ":")).encode() if payload is not None else None,
|
| 57 |
{"Authorization": "Bearer " + token, "Content-Type": "application/json"}, method=method,
|
| 58 |
)
|
| 59 |
with urllib.request.urlopen(request, timeout=30) as response:
|
| 60 |
return json.load(response)
|
| 61 |
|
| 62 |
|
| 63 |
+
def approved_art_rows(root, source_id=None):
|
| 64 |
+
"""Only the Lab SQLite inventory is read; no DemoService database driver."""
|
| 65 |
+
with sqlite3.connect((root / "inventory.sqlite3").resolve().as_uri() + "?mode=ro", uri=True) as db:
|
| 66 |
+
db.row_factory = sqlite3.Row
|
| 67 |
+
db.execute("PRAGMA query_only=ON")
|
| 68 |
+
db.execute("BEGIN")
|
| 69 |
+
query = """SELECT i.id,i.kind,i.label,i.city_id,i.city_name,i.design_id,i.record_path,
|
| 70 |
+
i.ai_json,i.geometry_ok,i.seoul_ok,i.status,a.accepted_at
|
| 71 |
+
FROM accepted a JOIN items i ON i.id=a.item_id
|
| 72 |
+
WHERE i.label IS a.label AND i.city_id IS a.city_id AND i.design_id IS a.design_id
|
| 73 |
+
AND NOT EXISTS (SELECT 1 FROM approval_revocations v WHERE v.item_id=i.id)"""
|
| 74 |
+
if source_id is not None:
|
| 75 |
+
query += " AND i.id=?"
|
| 76 |
+
rows = [dict(r) for r in db.execute(query + " ORDER BY i.id", (source_id,) if source_id is not None else ())]
|
| 77 |
+
total = db.execute("SELECT count(*) FROM accepted" + (" WHERE item_id=?" if source_id is not None else ""),
|
| 78 |
+
(source_id,) if source_id is not None else ()).fetchone()[0]
|
| 79 |
+
if total != len(rows):
|
| 80 |
+
raise RuntimeError("Invalid or revoked approval join")
|
| 81 |
+
return rows
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def verify_source_gpx(raw, expected):
|
| 85 |
+
try:
|
| 86 |
+
if not 0 < len(raw) <= 128 * 1024 or b"<!DOCTYPE" in raw.upper() or b"<!ENTITY" in raw.upper():
|
| 87 |
+
raise ValueError
|
| 88 |
+
document = ET.fromstring(raw)
|
| 89 |
+
tracks = [n for n in document.iter() if n.tag.rsplit("}", 1)[-1] == "trk"]
|
| 90 |
+
segments = [n for n in document.iter() if n.tag.rsplit("}", 1)[-1] == "trkseg"]
|
| 91 |
+
if document.tag.rsplit("}", 1)[-1] != "gpx" or len(tracks) != 1 or len(segments) != 1 or segments[0] not in list(tracks[0]):
|
| 92 |
+
raise ValueError
|
| 93 |
+
points = [(float(n.attrib["lat"]), float(n.attrib["lon"])) for n in segments[0]
|
| 94 |
+
if n.tag.rsplit("}", 1)[-1] == "trkpt"]
|
| 95 |
+
if not 12 <= len(points) <= 10000 or len(points) != len(expected):
|
| 96 |
+
raise ValueError
|
| 97 |
+
if (len(set(points)) < 4 or math.dist(points[0], points[-1]) > 1e-5
|
| 98 |
+
or not all(math.isfinite(lat) and math.isfinite(lon) and abs(lat) <= 90 and abs(lon) <= 180
|
| 99 |
+
and abs(lat - original[1]) <= 5.01e-8 and abs(lon - original[0]) <= 5.01e-8
|
| 100 |
+
for (lat, lon), original in zip(points, expected))):
|
| 101 |
+
raise ValueError
|
| 102 |
+
except (ValueError, TypeError, KeyError, IndexError, ET.ParseError):
|
| 103 |
+
raise RuntimeError("GPX does not match approved source GPS") from None
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def current_design_review(root, label, template):
|
| 107 |
+
item_id = hashlib.sha256(("design:" + label + ":" + template["design_digest"]).encode()).hexdigest()
|
| 108 |
+
with sqlite3.connect((root / "inventory.sqlite3").as_uri() + "?mode=ro", uri=True) as db:
|
| 109 |
+
db.execute("PRAGMA query_only=ON")
|
| 110 |
+
row = db.execute("SELECT status,ai_json FROM items WHERE id=? AND kind='design'", (item_id,)).fetchone()
|
| 111 |
+
return (row[0], json.loads(row[1] or "{}")) if row else ("unknown", {})
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def art_packets(root, probe_id=None, max_routes=None):
|
| 115 |
+
root = root.resolve()
|
| 116 |
+
rows = approved_art_rows(root)
|
| 117 |
+
if not rows:
|
| 118 |
+
raise RuntimeError("Empty approval list; refusing to synchronize")
|
| 119 |
+
if probe_id:
|
| 120 |
+
rows = [r for r in rows if r["id"] == probe_id]
|
| 121 |
+
if len(rows) != 1:
|
| 122 |
+
raise RuntimeError("Probe ID is not currently approved")
|
| 123 |
+
if max_routes is not None:
|
| 124 |
+
if max_routes <= 0:
|
| 125 |
+
raise RuntimeError("max_routes must be positive")
|
| 126 |
+
rows = rows[:max_routes]
|
| 127 |
+
packets, paths = {}, {}
|
| 128 |
+
finalized = (root / "completion.json").is_file()
|
| 129 |
+
with sqlite3.connect((root / "inventory.sqlite3").as_uri() + "?mode=ro", uri=True) as db:
|
| 130 |
+
db.execute("PRAGMA query_only=ON")
|
| 131 |
+
db.execute("BEGIN")
|
| 132 |
+
for row in rows:
|
| 133 |
+
source_id, label = row["id"], row["label"]
|
| 134 |
+
if (not re.fullmatch(r"[0-9a-f]{64}", source_id) or label not in {"dog", "heart", "tree", "tshirt", "candle", "pants", "star"}
|
| 135 |
+
or row["kind"] != "route" or row["status"] != "ai_pass" or row["geometry_ok"] != 1 or row["seoul_ok"] != 1):
|
| 136 |
+
raise RuntimeError("Approval state changed")
|
| 137 |
+
record_path = Path(row["record_path"])
|
| 138 |
+
record_raw = record_path.read_bytes()
|
| 139 |
+
record = json.loads(record_raw)
|
| 140 |
+
metadata, route = record["metadata"], record["route"]
|
| 141 |
+
design_path = Path(metadata.get("design_file") or ".")
|
| 142 |
+
if not design_path.is_file():
|
| 143 |
+
design_path = root / "designs" / label / (row["design_id"] + ".json")
|
| 144 |
+
design_raw = design_path.read_bytes()
|
| 145 |
+
design = json.loads(design_raw)
|
| 146 |
+
design_digest = hashlib.sha256(canonical(design).encode()).hexdigest()
|
| 147 |
+
if (design_digest != metadata.get("design_digest") or design.get("design_id") != row["design_id"]
|
| 148 |
+
or route.get("design_id") != row["design_id"] or str(design.get("version")) != str(route.get("design_version"))):
|
| 149 |
+
raise RuntimeError("Source design identity changed")
|
| 150 |
+
review = json.loads(row["ai_json"])
|
| 151 |
+
if (review.get("status") != "ai_pass" or review.get("expected_label") != label
|
| 152 |
+
or review.get("blind") is not True or review.get("qwen_performed") is not True
|
| 153 |
+
or not review.get("siglip_support_angles")):
|
| 154 |
+
raise RuntimeError("Missing route review evidence")
|
| 155 |
+
proof = route.get("verification", {})
|
| 156 |
+
if (not all(proof.get(k) is True for k in ("road_only", "graph_connectivity_verified", "access_checked", "directions_checked"))
|
| 157 |
+
or type(proof.get("air_length_m")) not in (int, float) or proof["air_length_m"] != 0 or not record.get("assessment", {}).get("success")
|
| 158 |
+
or not record.get("assessment", {}).get("route_valid")):
|
| 159 |
+
raise RuntimeError("Missing road or geometry evidence")
|
| 160 |
+
graph_hash = metadata.get("graph_sha256", "")
|
| 161 |
+
if not re.fullmatch(r"[0-9a-f]{64}", graph_hash):
|
| 162 |
+
raise RuntimeError("Missing graph digest")
|
| 163 |
+
length = route.get("length_m")
|
| 164 |
+
if type(length) not in (int, float) or not math.isfinite(length) or length <= 0:
|
| 165 |
+
raise RuntimeError("Invalid source road length")
|
| 166 |
+
design_item_id = hashlib.sha256(("design:" + label + ":" + design_digest).encode()).hexdigest()
|
| 167 |
+
design_review = db.execute("SELECT status,ai_json FROM items WHERE id=? AND kind='design'", (design_item_id,)).fetchone()
|
| 168 |
+
design_status = design_review[0] if design_review else "unknown"
|
| 169 |
+
design_evidence = json.loads(design_review[1] or "{}") if design_review else {}
|
| 170 |
+
gpx_path = root / "gpx" / label / (source_id + ".gpx")
|
| 171 |
+
gpx = gpx_path.read_bytes()
|
| 172 |
+
verify_source_gpx(gpx, route.get("gps", []))
|
| 173 |
+
packet = {"contract_version": 2, "label": label, "region_code": row["city_id"], "region_name": row["city_name"],
|
| 174 |
+
"sha256": hashlib.sha256(gpx).hexdigest(), "accepted_at": row["accepted_at"], "review_status": "ai_pass",
|
| 175 |
+
"ai_protocol": review["protocol"], "geometry_ok": True, "seoul_ok": True, "collection_finalized": finalized,
|
| 176 |
+
"gpx_base64": base64.b64encode(gpx).decode(),
|
| 177 |
+
"template": {"design_id": design["design_id"], "source_version": str(design["version"]),
|
| 178 |
+
"design_digest": design_digest, "snapshot": design, "design_ai_status": design_status,
|
| 179 |
+
"design_ai_evidence": design_evidence},
|
| 180 |
+
"route_ai_evidence": review, "road_verification": proof, "graph_sha256": graph_hash,
|
| 181 |
+
"distance_m": max(1, int(math.floor(length + .5))), "validator_version": "lab-approved-gpx-import-v1"}
|
| 182 |
+
packets[source_id] = packet
|
| 183 |
+
paths[source_id] = {str(p): hashlib.sha256(raw).hexdigest() for p, raw in
|
| 184 |
+
[(record_path, record_raw), (design_path, design_raw), (gpx_path, gpx)]}
|
| 185 |
+
return rows, packets, paths
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _validate_receipt(reply, source_id, packet):
|
| 189 |
+
try:
|
| 190 |
+
valid_ids = all(uuid.UUID(reply[k]) for k in ("art_template_id", "art_route_candidate_id"))
|
| 191 |
+
except (KeyError, ValueError, TypeError, AttributeError):
|
| 192 |
+
valid_ids = False
|
| 193 |
+
if (not valid_ids or reply.get("status") != "stored" or reply.get("contract_version") != 2
|
| 194 |
+
or reply.get("source_id") != source_id or reply.get("sha256") != packet["sha256"]
|
| 195 |
+
or reply.get("payload_sha256") != content_sha(packet)
|
| 196 |
+
or reply.get("design_ai_status") != packet["template"]["design_ai_status"]
|
| 197 |
+
or reply.get("route_validation_status") != "passed"):
|
| 198 |
+
raise RuntimeError("Missing or inconsistent art storage receipt")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def sync_v2(root, destination, token, state_path, probe_id=None, max_routes=None,
|
| 202 |
+
allow_updates=False, allow_reapproval=False, new_session=False):
|
| 203 |
+
root = root.resolve()
|
| 204 |
+
rows, packets, paths = art_packets(root, probe_id, max_routes)
|
| 205 |
+
manifest = {source_id: content_sha(packet) for source_id, packet in packets.items()}
|
| 206 |
+
manifest_hash = hashlib.sha256(canonical(manifest).encode()).hexdigest()
|
| 207 |
+
mode = "partial" if probe_id is not None or max_routes is not None else "full"
|
| 208 |
+
run_signature = hashlib.sha256(canonical([manifest_hash, mode]).encode()).hexdigest()
|
| 209 |
+
state_path.parent.mkdir(parents=True, exist_ok=True)
|
| 210 |
+
os.chmod(state_path.parent, 0o700)
|
| 211 |
+
endpoint = destination.rstrip("/")
|
| 212 |
+
with sqlite3.connect(state_path) as state:
|
| 213 |
+
os.chmod(state_path, 0o600)
|
| 214 |
+
state.row_factory = sqlite3.Row
|
| 215 |
+
state.execute("""CREATE TABLE IF NOT EXISTS http_v2_runs (
|
| 216 |
+
client_request_id TEXT PRIMARY KEY, signature TEXT NOT NULL, expected_generation INTEGER NOT NULL,
|
| 217 |
+
session_id TEXT, generation INTEGER, completed INTEGER NOT NULL DEFAULT 0)""")
|
| 218 |
+
state.execute("""CREATE TABLE IF NOT EXISTS http_v2_sent (
|
| 219 |
+
id TEXT PRIMARY KEY, signature TEXT NOT NULL, session_id TEXT NOT NULL,
|
| 220 |
+
template_id TEXT NOT NULL, candidate_id TEXT NOT NULL)""")
|
| 221 |
+
run = None if new_session else state.execute(
|
| 222 |
+
"SELECT * FROM http_v2_runs WHERE signature=? AND completed=0 ORDER BY rowid DESC LIMIT 1", (run_signature,)).fetchone()
|
| 223 |
+
if run is None:
|
| 224 |
+
remote = send(endpoint + "/sync-state", token, "GET", None)
|
| 225 |
+
if remote.get("contract_version") != 2 or type(remote.get("generation")) is not int or remote["generation"] < 0:
|
| 226 |
+
raise RuntimeError("Receiver does not support fenced v2 imports")
|
| 227 |
+
request_id = str(uuid.uuid4())
|
| 228 |
+
state.execute("INSERT INTO http_v2_runs(client_request_id,signature,expected_generation) VALUES (?,?,?)",
|
| 229 |
+
(request_id, run_signature, remote["generation"]))
|
| 230 |
+
state.commit() # Persist before the request so a lost response can be retried.
|
| 231 |
+
run = state.execute("SELECT * FROM http_v2_runs WHERE client_request_id=?", (request_id,)).fetchone()
|
| 232 |
+
session = send(endpoint + "/sync-sessions", token, "POST", {"contract_version": 2,
|
| 233 |
+
"client_request_id": run["client_request_id"], "expected_generation": run["expected_generation"],
|
| 234 |
+
"mode": mode, "manifest": manifest, "manifest_sha256": manifest_hash})
|
| 235 |
+
try:
|
| 236 |
+
session_id = str(uuid.UUID(session["session_id"]))
|
| 237 |
+
except (KeyError, ValueError, TypeError, AttributeError):
|
| 238 |
+
raise RuntimeError("Invalid sync session response") from None
|
| 239 |
+
if (session.get("contract_version") != 2 or type(session.get("generation")) is not int
|
| 240 |
+
or session["generation"] != run["expected_generation"] + 1 or session.get("mode") != mode
|
| 241 |
+
or session.get("manifest_sha256") != manifest_hash):
|
| 242 |
+
raise RuntimeError("Inconsistent sync session response")
|
| 243 |
+
state.execute("UPDATE http_v2_runs SET session_id=?,generation=? WHERE client_request_id=?",
|
| 244 |
+
(session_id, session["generation"], run["client_request_id"]))
|
| 245 |
+
state.commit()
|
| 246 |
+
uploaded = 0
|
| 247 |
+
for row in rows:
|
| 248 |
+
source_id = row["id"]
|
| 249 |
+
if approved_art_rows(root, source_id) != [row] or any(hashlib.sha256(Path(p).read_bytes()).hexdigest() != h
|
| 250 |
+
for p, h in paths[source_id].items()):
|
| 251 |
+
raise RuntimeError("Approval or source file changed before upload")
|
| 252 |
+
template = packets[source_id]["template"]
|
| 253 |
+
if current_design_review(root, row["label"], template) != (template["design_ai_status"], template["design_ai_evidence"]):
|
| 254 |
+
raise RuntimeError("Design review changed before upload")
|
| 255 |
+
packet = {**packets[source_id], "session_id": session_id, "generation": session["generation"],
|
| 256 |
+
"payload_sha256": manifest[source_id]}
|
| 257 |
+
previous = state.execute("SELECT * FROM http_v2_sent WHERE id=?", (source_id,)).fetchone()
|
| 258 |
+
if previous and previous["session_id"] == session_id and previous["signature"] == manifest[source_id]:
|
| 259 |
+
continue
|
| 260 |
+
if previous and allow_updates:
|
| 261 |
+
packet["previous_payload_sha256"] = previous["signature"]
|
| 262 |
+
if allow_reapproval:
|
| 263 |
+
packet["reapprove"] = True
|
| 264 |
+
received = send(endpoint + "/routes/" + source_id, token, "PUT", packet)
|
| 265 |
+
_validate_receipt(received, source_id, packet)
|
| 266 |
+
state.execute("""INSERT INTO http_v2_sent VALUES (?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET
|
| 267 |
+
signature=excluded.signature,session_id=excluded.session_id,
|
| 268 |
+
template_id=excluded.template_id,candidate_id=excluded.candidate_id""",
|
| 269 |
+
(source_id, manifest[source_id], session_id, received["art_template_id"], received["art_route_candidate_id"]))
|
| 270 |
+
state.commit()
|
| 271 |
+
uploaded += 1
|
| 272 |
+
current_rows, current_packets, _ = art_packets(root, probe_id, max_routes)
|
| 273 |
+
if current_rows != rows or {k: content_sha(v) for k, v in current_packets.items()} != manifest:
|
| 274 |
+
raise RuntimeError("Approval or metadata changed; snapshot not sent")
|
| 275 |
+
if mode == "full":
|
| 276 |
+
ids = sorted(manifest)
|
| 277 |
+
result = send(endpoint + "/approval-snapshot", token, "POST", {"contract_version": 2,
|
| 278 |
+
"session_id": session_id, "generation": session["generation"], "manifest_sha256": manifest_hash,
|
| 279 |
+
"ids": ids, "count": len(ids), "ids_sha256": hashlib.sha256("\n".join(ids).encode()).hexdigest(),
|
| 280 |
+
"collection_finalized": (root / "completion.json").is_file()})
|
| 281 |
+
if (result.get("contract_version") != 2 or result.get("session_id") != session_id
|
| 282 |
+
or result.get("generation") != session["generation"] or result.get("active_count") != len(ids)
|
| 283 |
+
or type(result.get("deactivated_count")) is not int):
|
| 284 |
+
raise RuntimeError("Invalid snapshot storage response")
|
| 285 |
+
else:
|
| 286 |
+
result = send(endpoint + "/sync-sessions/" + session_id + "/complete?generation=" + str(session["generation"]),
|
| 287 |
+
token, "POST", {})
|
| 288 |
+
if (result.get("session_id") != session_id or result.get("generation") != session["generation"]
|
| 289 |
+
or result.get("session_status") != "completed" or result.get("deactivated_count") != 0):
|
| 290 |
+
raise RuntimeError("Invalid partial completion response")
|
| 291 |
+
state.execute("UPDATE http_v2_runs SET completed=1 WHERE client_request_id=?", (run["client_request_id"],))
|
| 292 |
+
state.commit()
|
| 293 |
+
return {"contract_version": 2, "approved": len(rows), "uploaded": uploaded, "snapshot_sent": mode == "full",
|
| 294 |
+
"generation": session["generation"], "deactivated_count": result["deactivated_count"]}
|
| 295 |
+
|
| 296 |
+
|
| 297 |
def sync(root: Path, destination: str, token: str, state_path: Path,
|
| 298 |
probe_id: str | None = None) -> dict:
|
| 299 |
rows = approved_rows(root)
|
|
|
|
| 372 |
parser.add_argument("--state", type=Path, default=Path.home() / ".local/share/demoservice-sync/state.sqlite3")
|
| 373 |
parser.add_argument("--interval-seconds", type=int, default=0, help="Repeat after this delay; 0 runs once")
|
| 374 |
parser.add_argument("--probe-id", help="Send one approved GPX without an approval snapshot")
|
| 375 |
+
parser.add_argument("--contract-version", type=int, choices=[1, 2], default=1,
|
| 376 |
+
help="Opt in to fenced art imports with 2; existing callers remain on 1")
|
| 377 |
+
parser.add_argument("--max-routes", type=int, help="v2 partial import limit; never revokes")
|
| 378 |
+
parser.add_argument("--allow-updates", action="store_true", help="v2 compare-and-swap using a previous receipt")
|
| 379 |
+
parser.add_argument("--allow-reapproval", action="store_true", help="v2 explicit reapproval with a later accepted_at")
|
| 380 |
+
parser.add_argument("--new-session", action="store_true", help="Abandon local pending session and start from current source approvals")
|
| 381 |
+
parser.add_argument("--check-only", action="store_true", help="v2 local preparation only; no token read or HTTP")
|
| 382 |
args = parser.parse_args()
|
| 383 |
+
if args.check_only:
|
| 384 |
+
rows, packets, _ = art_packets(args.root, args.probe_id, args.max_routes)
|
| 385 |
+
print(json.dumps({"approved": len(rows), "templates": len({p['template']['design_digest'] for p in packets.values()}),
|
| 386 |
+
"http_requests": 0}, ensure_ascii=False))
|
| 387 |
+
return
|
| 388 |
token = args.token_file.read_text(encoding="utf-8").strip()
|
| 389 |
if not token:
|
| 390 |
raise SystemExit("Empty receiver token")
|
|
|
|
| 392 |
parser.error("--interval-seconds must be nonnegative")
|
| 393 |
if args.probe_id and args.interval_seconds:
|
| 394 |
parser.error("--probe-id cannot be used with --interval-seconds")
|
| 395 |
+
if args.contract_version == 1 and (args.max_routes or args.allow_updates or args.allow_reapproval or args.new_session):
|
| 396 |
+
parser.error("v2 options require --contract-version 2")
|
| 397 |
+
from urllib.parse import urlsplit
|
| 398 |
+
endpoint = urlsplit(args.url)
|
| 399 |
+
if endpoint.username or endpoint.password or endpoint.query or endpoint.fragment or not endpoint.hostname:
|
| 400 |
+
parser.error("Use an API URL without embedded credentials, query or fragment")
|
| 401 |
+
if endpoint.scheme != "https" and not (endpoint.scheme == "http" and endpoint.hostname in {"localhost", "127.0.0.1", "::1"}):
|
| 402 |
+
parser.error("Use HTTPS, or loopback HTTP for synthetic tests")
|
| 403 |
+
import fcntl
|
| 404 |
+
args.state.parent.mkdir(parents=True, exist_ok=True)
|
| 405 |
+
with args.state.with_suffix(args.state.suffix + ".lock").open("a") as lock:
|
| 406 |
try:
|
| 407 |
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 408 |
+
except BlockingIOError:
|
| 409 |
+
raise SystemExit("A sender already owns this state file") from None
|
| 410 |
+
while True:
|
| 411 |
+
try:
|
| 412 |
+
if args.contract_version == 2:
|
| 413 |
+
result = sync_v2(args.root, args.url, token, args.state, args.probe_id, args.max_routes,
|
| 414 |
+
args.allow_updates, args.allow_reapproval, args.new_session)
|
| 415 |
+
else:
|
| 416 |
+
result = sync(args.root, args.url, token, args.state, args.probe_id)
|
| 417 |
+
print(json.dumps(result, ensure_ascii=False), flush=True)
|
| 418 |
+
except Exception as exc:
|
| 419 |
+
if not args.interval_seconds:
|
| 420 |
+
raise SystemExit(f"Sync failed: {type(exc).__name__}; details suppressed") from None
|
| 421 |
+
print(f"Sync retry scheduled after {type(exc).__name__}; details suppressed", flush=True)
|
| 422 |
if not args.interval_seconds:
|
| 423 |
+
break
|
| 424 |
+
# Only the first run is explicitly forced; subsequent runs resume normally.
|
| 425 |
+
args.new_session = False
|
| 426 |
+
time.sleep(args.interval_seconds)
|
|
|
|
| 427 |
|
| 428 |
|
| 429 |
if __name__ == "__main__":
|