Spaces:
Running
Running
Deploy complete BrainAPI source 42cde77950db
Browse filesAll tracked files from Git commit 42cde77950db2e24a1b7acbc3477274bd9cad06a; required Docker inputs checked before upload.
- .env.example +4 -3
- .gitignore +1 -2
- deploy/README.md +38 -2
- deploy/env.heavy.example +4 -3
- deploy/env.light.example +4 -3
- deploy/env.public-demo.example +2 -2
- entrypoint.sh +6 -19
- scripts/qualify_brain_registry.py +171 -0
- scripts/seed_public_demo.py +16 -3
- src/adapters/data.py +11 -1
- src/config.py +2 -20
- src/lib/mongo/client.py +7 -1
- src/lib/postgresql/_provisioning.py +50 -1
- src/lib/postgresql/data.py +34 -38
- src/lib/postgresql/graph_store.py +8 -5
- src/lib/postgresql/vectors.py +7 -2
- src/services/api/controllers/system.py +3 -4
- src/services/api/errors.py +20 -0
- src/services/api/middlewares/auth.py +1 -1
- src/services/api/middlewares/brains.py +25 -68
- src/services/api/routes/public.py +5 -1
- src/services/api/routes/system.py +4 -4
- src/services/brain_identity.py +33 -0
- src/services/brain_lifecycle.py +8 -23
- src/services/mcp/utils.py +5 -0
- src/services/workspaces.py +6 -1
- tests/integration/test_brain_registry_contract.py +27 -0
- tests/test_agent_api.py +0 -1
- tests/test_agent_readiness_api.py +3 -3
- tests/test_brain_lifecycle.py +1 -1
- tests/test_brain_registry_contract.py +341 -0
- tests/test_embeddinggemma2.py +2 -0
- tests/test_graph_upsert_contract.py +2 -1
- tests/test_retrieve_auth_isolation.py +1 -1
- tests/test_retrieve_http_contract.py +8 -0
- tests/test_search_bm25.py +9 -0
- tests/test_workspace_facade.py +0 -1
- tests/test_workspace_plugin_contract.py +0 -1
.env.example
CHANGED
|
@@ -102,13 +102,14 @@ PUBLIC_BASE_URL="http://localhost:8000"
|
|
| 102 |
BRAINPAT_TOKEN="your_token"
|
| 103 |
|
| 104 |
# MultiBrain
|
| 105 |
-
#
|
| 106 |
-
|
|
|
|
| 107 |
|
| 108 |
# Production ingress and plugin startup policy.
|
| 109 |
CORS_ALLOWED_ORIGINS=""
|
| 110 |
PLUGIN_FAILURE_POLICY="fail"
|
| 111 |
-
#
|
| 112 |
DEFAULT_BRAIN_FALLBACK="true"
|
| 113 |
|
| 114 |
# Optional anonymous, read-only documentation search demo (disabled by default)
|
|
|
|
| 102 |
BRAINPAT_TOKEN="your_token"
|
| 103 |
|
| 104 |
# MultiBrain
|
| 105 |
+
# Legacy flag; API requests never provision brains implicitly.
|
| 106 |
+
# Create explicitly through POST /system/brains or POST /system/workspaces.
|
| 107 |
+
BRAIN_CREATION_ALLOWED="false"
|
| 108 |
|
| 109 |
# Production ingress and plugin startup policy.
|
| 110 |
CORS_ALLOWED_ORIGINS=""
|
| 111 |
PLUGIN_FAILURE_POLICY="fail"
|
| 112 |
+
# Resolve a missing scope to an already-registered default brain; never creates it
|
| 113 |
DEFAULT_BRAIN_FALLBACK="true"
|
| 114 |
|
| 115 |
# Optional anonymous, read-only documentation search demo (disabled by default)
|
.gitignore
CHANGED
|
@@ -17,8 +17,7 @@ gcp_credentials.json
|
|
| 17 |
mcp-stdio-http-bridge/target/
|
| 18 |
mcp-stdio-http-bridge/dist/
|
| 19 |
|
| 20 |
-
plugins/*
|
| 21 |
-
!plugins/readme.txt
|
| 22 |
|
| 23 |
|
| 24 |
.git
|
|
|
|
| 17 |
mcp-stdio-http-bridge/target/
|
| 18 |
mcp-stdio-http-bridge/dist/
|
| 19 |
|
| 20 |
+
plugins/**
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
.git
|
deploy/README.md
CHANGED
|
@@ -117,10 +117,46 @@ docker compose --env-file /etc/brainapi/light.env -f deploy/docker-compose.light
|
|
| 117 |
The seed command downloads the published V2 documentation, records its SHA-256
|
| 118 |
marker, skips an already-current corpus, and polls the asynchronous ingestion
|
| 119 |
task. It requires the private system `BRAINPAT_TOKEN` inside the deployment; do
|
| 120 |
-
not expose that token through the gateway.
|
| 121 |
-
`BRAIN_CREATION_ALLOWED`
|
| 122 |
|
| 123 |
Expose the API origin as `api.brain-api.dev`, with TLS, and put the Cloudflare
|
| 124 |
gateway in front of root-domain `/api/*`. The application itself deliberately
|
| 125 |
does not implement client rate limiting; the checked-in gateway owns the
|
| 126 |
30-request/60-second demo limit.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
The seed command downloads the published V2 documentation, records its SHA-256
|
| 118 |
marker, skips an already-current corpus, and polls the asynchronous ingestion
|
| 119 |
task. It requires the private system `BRAINPAT_TOKEN` inside the deployment; do
|
| 120 |
+
not expose that token through the gateway. The seed command explicitly creates
|
| 121 |
+
the brain; `BRAIN_CREATION_ALLOWED` does not enable implicit API provisioning.
|
| 122 |
|
| 123 |
Expose the API origin as `api.brain-api.dev`, with TLS, and put the Cloudflare
|
| 124 |
gateway in front of root-domain `/api/*`. The application itself deliberately
|
| 125 |
does not implement client rate limiting; the checked-in gateway owns the
|
| 126 |
30-request/60-second demo limit.
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
## Brain registry contract
|
| 130 |
+
|
| 131 |
+
The persisted registry is authoritative. Retrieval, query, agent, and MCP access
|
| 132 |
+
cannot provision an unknown brain, even with `BRAIN_CREATION_ALLOWED=true`.
|
| 133 |
+
Explicit unknown API scopes return `404 BRAIN_NOT_FOUND`. A missing scope can
|
| 134 |
+
resolve to `default` only when `DEFAULT_BRAIN_FALLBACK=true` and that brain is
|
| 135 |
+
already registered. Create brains with `POST /system/brains` or
|
| 136 |
+
`POST /system/workspaces`; duplicate brain creation returns
|
| 137 |
+
`409 BRAIN_ALREADY_EXISTS`, including concurrent inserts.
|
| 138 |
+
|
| 139 |
+
Brain IDs accept 1–128 ASCII letters, digits, hyphens, and underscores, starting
|
| 140 |
+
with a letter or digit. `system` is reserved (case insensitive). The same rules
|
| 141 |
+
apply to create, clear, and delete; workspace slugs retain their narrower syntax.
|
| 142 |
+
`/system/brains-list` returns persisted registry identities only. Orphan physical
|
| 143 |
+
databases never appear as registered brains or acquire generated PATs.
|
| 144 |
+
|
| 145 |
+
Clear preserves registered identity and physical storage. Explicit delete also
|
| 146 |
+
removes the PostgreSQL database, including a named orphan without a registry
|
| 147 |
+
row. These are destructive operations: target only an intended identifier.
|
| 148 |
+
|
| 149 |
+
Run the deterministic gate and the opt-in real integration gate:
|
| 150 |
+
|
| 151 |
+
```bash
|
| 152 |
+
PYTHONPATH=.:tests python -m pytest -q tests/test_brain_registry_contract.py tests/test_brain_lifecycle.py
|
| 153 |
+
BRAINAPI_REGISTRY_TEST_CONTAINER=<disposable-local-container> \
|
| 154 |
+
PYTHONPATH=.:tests python -m pytest -q tests/integration/test_brain_registry_contract.py
|
| 155 |
+
```
|
| 156 |
+
|
| 157 |
+
The integration gate requires a disposable local HF CPU container with the
|
| 158 |
+
candidate source and its normal entrypoint environment. It invokes
|
| 159 |
+
`bash /app/entrypoint.sh --qualify-registry`, compares actual PostgreSQL catalogs
|
| 160 |
+
and application Redis state around unknown GET/search/context requests, tests
|
| 161 |
+
concurrent storage inserts, and deletes only its own collision-checked temporary
|
| 162 |
+
namespaces. It does not clean production probe artifacts or other existing brains.
|
deploy/env.heavy.example
CHANGED
|
@@ -104,9 +104,10 @@ MONGO_SYSTEM_DATABASE="system"
|
|
| 104 |
BRAINPAT_TOKEN="your_token"
|
| 105 |
|
| 106 |
# MultiBrain
|
| 107 |
-
#
|
| 108 |
-
|
| 109 |
-
|
|
|
|
| 110 |
DEFAULT_BRAIN_FALLBACK="true"
|
| 111 |
|
| 112 |
# Optional anonymous, read-only documentation search demo (disabled by default)
|
|
|
|
| 104 |
BRAINPAT_TOKEN="your_token"
|
| 105 |
|
| 106 |
# MultiBrain
|
| 107 |
+
# Legacy flag; API requests never provision brains implicitly.
|
| 108 |
+
# Create explicitly through POST /system/brains or POST /system/workspaces.
|
| 109 |
+
BRAIN_CREATION_ALLOWED="false"
|
| 110 |
+
# Resolve a missing scope to an already-registered default brain; never creates it
|
| 111 |
DEFAULT_BRAIN_FALLBACK="true"
|
| 112 |
|
| 113 |
# Optional anonymous, read-only documentation search demo (disabled by default)
|
deploy/env.light.example
CHANGED
|
@@ -102,9 +102,10 @@ MONGO_SYSTEM_DATABASE="system"
|
|
| 102 |
BRAINPAT_TOKEN="your_token"
|
| 103 |
|
| 104 |
# MultiBrain
|
| 105 |
-
#
|
| 106 |
-
|
| 107 |
-
|
|
|
|
| 108 |
DEFAULT_BRAIN_FALLBACK="true"
|
| 109 |
|
| 110 |
# Optional anonymous, read-only documentation search demo (disabled by default)
|
|
|
|
| 102 |
BRAINPAT_TOKEN="your_token"
|
| 103 |
|
| 104 |
# MultiBrain
|
| 105 |
+
# Legacy flag; API requests never provision brains implicitly.
|
| 106 |
+
# Create explicitly through POST /system/brains or POST /system/workspaces.
|
| 107 |
+
BRAIN_CREATION_ALLOWED="false"
|
| 108 |
+
# Resolve a missing scope to an already-registered default brain; never creates it
|
| 109 |
DEFAULT_BRAIN_FALLBACK="true"
|
| 110 |
|
| 111 |
# Optional anonymous, read-only documentation search demo (disabled by default)
|
deploy/env.public-demo.example
CHANGED
|
@@ -12,5 +12,5 @@ SEARCH_FUSION="rrf"
|
|
| 12 |
SEARCH_BM25_K1="1.2"
|
| 13 |
SEARCH_BM25_B="0.75"
|
| 14 |
|
| 15 |
-
#
|
| 16 |
-
BRAIN_CREATION_ALLOWED="
|
|
|
|
| 12 |
SEARCH_BM25_K1="1.2"
|
| 13 |
SEARCH_BM25_B="0.75"
|
| 14 |
|
| 15 |
+
# The seed command explicitly provisions agentdemo through POST /system/brains.
|
| 16 |
+
BRAIN_CREATION_ALLOWED="false"
|
entrypoint.sh
CHANGED
|
@@ -11,12 +11,12 @@ export PIPELINE_MODE="${PIPELINE_MODE:-lightweight}"
|
|
| 11 |
export MODELS_MODE="${MODELS_MODE:-local}"
|
| 12 |
export LLM_SMALL_PROVIDER="${LLM_SMALL_PROVIDER:-ollama}"
|
| 13 |
export LLM_LARGE_PROVIDER="${LLM_LARGE_PROVIDER:-ollama}"
|
| 14 |
-
export EMBEDDINGS_PROVIDER="ollama"
|
| 15 |
export OLLAMA_HOST="${OLLAMA_HOST:-127.0.0.1}"
|
| 16 |
export OLLAMA_PORT="${OLLAMA_PORT:-11434}"
|
| 17 |
export OLLAMA_MODELS="${OLLAMA_MODELS:-/opt/ollama-models}"
|
| 18 |
export OLLAMA_NO_CLOUD="${OLLAMA_NO_CLOUD:-true}"
|
| 19 |
-
export OLLAMA_EMBEDDING_MODEL="embeddinggemma-2:270m"
|
| 20 |
export OLLAMA_NUM_PARALLEL="${OLLAMA_NUM_PARALLEL:-1}"
|
| 21 |
export OLLAMA_MAX_LOADED_MODELS="${OLLAMA_MAX_LOADED_MODELS:-1}"
|
| 22 |
export OLLAMA_NUM_THREADS="${OLLAMA_NUM_THREADS:-2}"
|
|
@@ -74,26 +74,13 @@ export REDIS_PASSWORD
|
|
| 74 |
export CELERY_BROKER_URL="redis://:${REDIS_PASSWORD}@127.0.0.1:6379/0"
|
| 75 |
export CELERY_RESULT_BACKEND="redis://:${REDIS_PASSWORD}@127.0.0.1:6379/0"
|
| 76 |
|
| 77 |
-
echo "========== Effective embedding configuration =========="
|
| 78 |
-
printf 'EMBEDDINGS_PROVIDER=%q\n' "${EMBEDDINGS_PROVIDER:-}"
|
| 79 |
-
printf 'OLLAMA_EMBEDDING_MODEL=%q\n' "${OLLAMA_EMBEDDING_MODEL:-}"
|
| 80 |
-
printf 'EMBEDDINGS_LOCAL_MODEL=%q\n' "${EMBEDDINGS_LOCAL_MODEL:-}"
|
| 81 |
-
printf 'EMBEDDING_PROFILE=%q\n' "${EMBEDDING_PROFILE:-}"
|
| 82 |
-
printf 'EMBEDDING_NATIVE_DIMENSION=%q\n' "${EMBEDDING_NATIVE_DIMENSION:-}"
|
| 83 |
-
printf 'EMBEDDING_MRL_DIMENSION=%q\n' "${EMBEDDING_MRL_DIMENSION:-}"
|
| 84 |
-
printf 'EMBEDDING_NODES_DIMENSION=%q\n' "${EMBEDDING_NODES_DIMENSION:-}"
|
| 85 |
-
printf 'EMBEDDING_TRIPLETS_DIMENSION=%q\n' "${EMBEDDING_TRIPLETS_DIMENSION:-}"
|
| 86 |
-
printf 'EMBEDDING_OBSERVATIONS_DIMENSION=%q\n' "${EMBEDDING_OBSERVATIONS_DIMENSION:-}"
|
| 87 |
-
printf 'EMBEDDING_DATA_DIMENSION=%q\n' "${EMBEDDING_DATA_DIMENSION:-}"
|
| 88 |
-
printf 'EMBEDDING_RELATIONSHIPS_DIMENSION=%q\n' "${EMBEDDING_RELATIONSHIPS_DIMENSION:-}"
|
| 89 |
-
echo "========================================================"
|
| 90 |
-
|
| 91 |
# docker exec starts with the image environment, not this script's exports.
|
| 92 |
# Load the same runtime defaults/credentials for qualification without restarting
|
| 93 |
# or reinitializing the already-running application services.
|
| 94 |
-
|
| 95 |
-
exec /app/.venv/bin/python /app/scripts/qualify_hf_embeddings.py
|
| 96 |
-
|
|
|
|
| 97 |
|
| 98 |
# Print non-secret startup diagnostics
|
| 99 |
echo "=================================================="
|
|
|
|
| 11 |
export MODELS_MODE="${MODELS_MODE:-local}"
|
| 12 |
export LLM_SMALL_PROVIDER="${LLM_SMALL_PROVIDER:-ollama}"
|
| 13 |
export LLM_LARGE_PROVIDER="${LLM_LARGE_PROVIDER:-ollama}"
|
| 14 |
+
export EMBEDDINGS_PROVIDER="${EMBEDDINGS_PROVIDER:-ollama}"
|
| 15 |
export OLLAMA_HOST="${OLLAMA_HOST:-127.0.0.1}"
|
| 16 |
export OLLAMA_PORT="${OLLAMA_PORT:-11434}"
|
| 17 |
export OLLAMA_MODELS="${OLLAMA_MODELS:-/opt/ollama-models}"
|
| 18 |
export OLLAMA_NO_CLOUD="${OLLAMA_NO_CLOUD:-true}"
|
| 19 |
+
export OLLAMA_EMBEDDING_MODEL="${OLLAMA_EMBEDDING_MODEL:-embeddinggemma-2:270m}"
|
| 20 |
export OLLAMA_NUM_PARALLEL="${OLLAMA_NUM_PARALLEL:-1}"
|
| 21 |
export OLLAMA_MAX_LOADED_MODELS="${OLLAMA_MAX_LOADED_MODELS:-1}"
|
| 22 |
export OLLAMA_NUM_THREADS="${OLLAMA_NUM_THREADS:-2}"
|
|
|
|
| 74 |
export CELERY_BROKER_URL="redis://:${REDIS_PASSWORD}@127.0.0.1:6379/0"
|
| 75 |
export CELERY_RESULT_BACKEND="redis://:${REDIS_PASSWORD}@127.0.0.1:6379/0"
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
# docker exec starts with the image environment, not this script's exports.
|
| 78 |
# Load the same runtime defaults/credentials for qualification without restarting
|
| 79 |
# or reinitializing the already-running application services.
|
| 80 |
+
case "${1:-}" in
|
| 81 |
+
--qualify) exec /app/.venv/bin/python /app/scripts/qualify_hf_embeddings.py ;;
|
| 82 |
+
--qualify-registry) exec /app/.venv/bin/python /app/scripts/qualify_brain_registry.py ;;
|
| 83 |
+
esac
|
| 84 |
|
| 85 |
# Print non-secret startup diagnostics
|
| 86 |
echo "=================================================="
|
scripts/qualify_brain_registry.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Destructive only to unique disposable test namespaces; run via entrypoint --qualify-registry.
|
| 2 |
+
|
| 3 |
+
Exercises actual HTTP controllers, PostgreSQL catalogs, Redis, graph/vector clients,
|
| 4 |
+
and concurrent registry inserts. Never targets a preexisting brain or external data.
|
| 5 |
+
"""
|
| 6 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import sys
|
| 11 |
+
from uuid import uuid4
|
| 12 |
+
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 14 |
+
import httpx
|
| 15 |
+
|
| 16 |
+
from src.core.instances import cache_adapter, data_adapter, graph_adapter, vector_store_adapter
|
| 17 |
+
from src.lib.postgresql._naming import brain_db_name
|
| 18 |
+
from src.lib.postgresql import _provisioning as pg
|
| 19 |
+
from src.services.brain_identity import BrainAlreadyExistsError, BrainNotFoundError
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def check(condition, message):
|
| 23 |
+
if not condition:
|
| 24 |
+
raise RuntimeError(message)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def identities(api):
|
| 28 |
+
response = api.get('/system/brains-list')
|
| 29 |
+
response.raise_for_status()
|
| 30 |
+
return sorted((b['name_key'], b['id']) for b in response.json())
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def storage_snapshot():
|
| 34 |
+
with pg.borrow(pg.get_system_pool()) as conn, conn.cursor() as cursor:
|
| 35 |
+
cursor.execute('SELECT id, name_key, document FROM data_brains ORDER BY name_key')
|
| 36 |
+
registry = cursor.fetchall()
|
| 37 |
+
cursor.execute('SELECT document FROM data_workspaces ORDER BY id')
|
| 38 |
+
workspaces = cursor.fetchall()
|
| 39 |
+
with pg.borrow(pg.get_pool(pg._maintenance_dbname())) as conn, conn.cursor() as cursor:
|
| 40 |
+
cursor.execute('SELECT datname FROM pg_database ORDER BY datname')
|
| 41 |
+
databases = cursor.fetchall()
|
| 42 |
+
# Include values, not just key names. Unrelated Celery heartbeat bookkeeping
|
| 43 |
+
# is outside application namespaces and excluded from this comparison.
|
| 44 |
+
redis = cache_adapter.cache.client
|
| 45 |
+
cache = {key: redis.dump(key) for pattern in ('system:brain*', 'codex*')
|
| 46 |
+
for key in redis.scan_iter(match=pattern)}
|
| 47 |
+
return registry, workspaces, databases, cache
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def read(api, endpoint, brain_id):
|
| 51 |
+
if endpoint == 'entities':
|
| 52 |
+
return api.get('/retrieve/entities', params={'brain_id': brain_id})
|
| 53 |
+
return api.post('/retrieve/' + endpoint, json={
|
| 54 |
+
'brain_id': brain_id, 'query' if endpoint == 'search' else 'text': 'registry isolation probe',
|
| 55 |
+
})
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def qualify():
|
| 59 |
+
suffix = uuid4().hex[:12]
|
| 60 |
+
unknown = 'codex_unknown_' + suffix
|
| 61 |
+
names = ['codex-test-' + suffix, 'codex_test_' + suffix]
|
| 62 |
+
orphan = 'codex_orphan_' + suffix
|
| 63 |
+
race = 'codex_race_' + suffix
|
| 64 |
+
# Cleanup is limited to namespaces proven absent before this test.
|
| 65 |
+
temporary = names + [orphan, race]
|
| 66 |
+
created = set()
|
| 67 |
+
results = {}
|
| 68 |
+
api_url = os.getenv('QUALIFY_API_URL', 'http://127.0.0.1:7860')
|
| 69 |
+
with httpx.Client(base_url=api_url, headers={'BrainPAT': os.environ['BRAINPAT_TOKEN']},
|
| 70 |
+
timeout=120, trust_env=False) as api:
|
| 71 |
+
before_ids = identities(api)
|
| 72 |
+
before = storage_snapshot()
|
| 73 |
+
for name in temporary + [unknown]:
|
| 74 |
+
check(data_adapter.get_brain(name) is None, 'Test namespace collision')
|
| 75 |
+
check(brain_db_name(name) not in {r[0] for r in before[2]}, 'Test storage collision')
|
| 76 |
+
try:
|
| 77 |
+
for endpoint in ('entities', 'search', 'context'):
|
| 78 |
+
response = read(api, endpoint, unknown)
|
| 79 |
+
check(response.status_code == 404, 'Unknown ' + endpoint + ' did not return 404')
|
| 80 |
+
check(response.json()['error']['code'] == 'BRAIN_NOT_FOUND', 'Wrong missing-brain code')
|
| 81 |
+
check(storage_snapshot() == before, 'Unknown read changed registry/workspaces/databases/cache')
|
| 82 |
+
check(identities(api) == before_ids, 'Unknown read changed public identities')
|
| 83 |
+
results['unknown_' + endpoint] = '404 BRAIN_NOT_FOUND; storage/cache unchanged'
|
| 84 |
+
|
| 85 |
+
# Defense in depth: bypass HTTP and hit each real storage read path.
|
| 86 |
+
data = data_adapter.data
|
| 87 |
+
graph = graph_adapter.graph._store
|
| 88 |
+
vectors = vector_store_adapter.vector_store
|
| 89 |
+
backend_before = (set(data._initialized_brains), set(graph._schema_ready),
|
| 90 |
+
dict(graph._brains), set(vectors._initialized_stores),
|
| 91 |
+
set(vectors._brain_extensions_ready))
|
| 92 |
+
for operation in (
|
| 93 |
+
lambda: data.get_text_chunks(brain_id=unknown),
|
| 94 |
+
lambda: graph.get_brain(unknown),
|
| 95 |
+
lambda: vectors.search_vectors([1.0] + [0.0] * 511, brain_id=unknown, store='data'),
|
| 96 |
+
):
|
| 97 |
+
try:
|
| 98 |
+
operation()
|
| 99 |
+
except BrainNotFoundError:
|
| 100 |
+
pass
|
| 101 |
+
else:
|
| 102 |
+
raise RuntimeError('Unknown direct backend read accepted')
|
| 103 |
+
check(storage_snapshot() == before, 'Direct backend read changed persistent state')
|
| 104 |
+
check(backend_before == (set(data._initialized_brains), set(graph._schema_ready),
|
| 105 |
+
dict(graph._brains), set(vectors._initialized_stores),
|
| 106 |
+
set(vectors._brain_extensions_ready)), 'Unknown backend read populated a cache')
|
| 107 |
+
results['direct_backend_reads'] = 'data/graph/vector rejected; catalogs and caches unchanged'
|
| 108 |
+
|
| 109 |
+
for name in names:
|
| 110 |
+
created.add(name)
|
| 111 |
+
response = api.post('/system/brains', json={'brain_id': name})
|
| 112 |
+
check(response.status_code == 200, 'Explicit create failed')
|
| 113 |
+
identity = (response.json()['name_key'], response.json()['id'])
|
| 114 |
+
duplicate = api.post('/system/brains', json={'brain_id': name})
|
| 115 |
+
check(duplicate.status_code == 409 and duplicate.json()['error']['code'] == 'BRAIN_ALREADY_EXISTS', 'Duplicate was not conflict')
|
| 116 |
+
check(identity in identities(api) and identities(api) == identities(api), 'Identity changed')
|
| 117 |
+
for endpoint in ('entities', 'search', 'context'):
|
| 118 |
+
response = read(api, endpoint, name)
|
| 119 |
+
check(response.status_code == 200, 'Known ' + endpoint + ' failed: ' + str(response.status_code))
|
| 120 |
+
clear = api.post('/system/brains/' + name + '/clear')
|
| 121 |
+
check(clear.status_code == 200, 'Clear rejected supported identifier')
|
| 122 |
+
check(identity in identities(api), 'Clear changed registry identity')
|
| 123 |
+
response = api.delete('/system/brains/' + name)
|
| 124 |
+
check(response.status_code == 200, 'Delete rejected supported identifier')
|
| 125 |
+
check(data_adapter.get_brain(name) is None, 'Delete retained registry')
|
| 126 |
+
check(brain_db_name(name) not in {r[0] for r in storage_snapshot()[2]}, 'Delete retained physical storage')
|
| 127 |
+
results['known_reads_identifiers_duplicates'] = 'GET/search/context 200; stable IDs; duplicate 409; clear/delete hyphen and underscore passed'
|
| 128 |
+
|
| 129 |
+
# No preflight lookup: exercise real concurrent storage uniqueness handling.
|
| 130 |
+
created.add(race)
|
| 131 |
+
def create_concurrently(_):
|
| 132 |
+
try:
|
| 133 |
+
data_adapter.create_brain(race)
|
| 134 |
+
return 'created'
|
| 135 |
+
except BrainAlreadyExistsError:
|
| 136 |
+
return 'conflict'
|
| 137 |
+
with ThreadPoolExecutor(max_workers=2) as executor:
|
| 138 |
+
outcomes = sorted(executor.map(create_concurrently, range(2)))
|
| 139 |
+
check(outcomes == ['conflict', 'created'], 'Storage uniqueness race was not a domain conflict')
|
| 140 |
+
results['storage_duplicate_race'] = 'one create, one domain conflict'
|
| 141 |
+
|
| 142 |
+
# Create only a physical database, without ever creating a registry row.
|
| 143 |
+
created.add(orphan)
|
| 144 |
+
pg.ensure_database_exists(brain_db_name(orphan))
|
| 145 |
+
check(data_adapter.get_brain(orphan) is None, 'Orphan unexpectedly registered')
|
| 146 |
+
check(all(n != orphan for n, _ in identities(api)), 'Orphan exposed as registered brain')
|
| 147 |
+
def orphan_schema():
|
| 148 |
+
with pg.borrow(pg.get_pool(brain_db_name(orphan))) as conn, conn.cursor() as cur:
|
| 149 |
+
cur.execute("SELECT schemaname, tablename FROM pg_tables ORDER BY schemaname, tablename")
|
| 150 |
+
return cur.fetchall()
|
| 151 |
+
orphan_before = orphan_schema()
|
| 152 |
+
for endpoint in ('entities', 'search', 'context'):
|
| 153 |
+
response = read(api, endpoint, orphan)
|
| 154 |
+
check(response.status_code == 404, 'Orphan storage authorized an unknown read')
|
| 155 |
+
check(orphan_schema() == orphan_before, 'Orphan read created tables')
|
| 156 |
+
response = api.delete('/system/brains/' + orphan)
|
| 157 |
+
check(response.status_code == 200 and response.json()['existed'] is False, 'Orphan cleanup failed')
|
| 158 |
+
check(brain_db_name(orphan) not in {r[0] for r in storage_snapshot()[2]}, 'Orphan database remains')
|
| 159 |
+
check(data_adapter.get_brain(orphan) is None, 'Orphan cleanup created a registry row')
|
| 160 |
+
results['orphan_cleanup'] = 'physical database removed without registry creation'
|
| 161 |
+
finally:
|
| 162 |
+
for name in sorted(created):
|
| 163 |
+
response = api.delete('/system/brains/' + name)
|
| 164 |
+
check(response.status_code == 200, 'Temporary brain cleanup failed')
|
| 165 |
+
check(identities(api) == before_ids, 'Temporary qualification changed preexisting brain identities')
|
| 166 |
+
results['result'] = 'passed'
|
| 167 |
+
print(json.dumps(results, sort_keys=True))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
if __name__ == '__main__':
|
| 171 |
+
qualify()
|
scripts/seed_public_demo.py
CHANGED
|
@@ -10,6 +10,10 @@ import sys
|
|
| 10 |
import time
|
| 11 |
|
| 12 |
import requests
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
|
| 15 |
DEFAULT_DOCS_URL = "https://brainapi.lumen-labs.ai/docs/llms-full.txt"
|
|
@@ -27,8 +31,10 @@ def main() -> int:
|
|
| 27 |
if not token:
|
| 28 |
print("BRAINPAT_TOKEN is required", file=sys.stderr)
|
| 29 |
return 2
|
| 30 |
-
|
| 31 |
-
|
|
|
|
|
|
|
| 32 |
return 2
|
| 33 |
|
| 34 |
api_url = args.api_url.rstrip("/")
|
|
@@ -48,7 +54,14 @@ def main() -> int:
|
|
| 48 |
if existing.status_code == 200 and existing.json().get("total", 0) > 0:
|
| 49 |
print(f"Public demo brain {args.brain!r} is already current ({digest[:12]}).")
|
| 50 |
return 0
|
| 51 |
-
if existing.status_code
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
existing.raise_for_status()
|
| 53 |
|
| 54 |
payload = {
|
|
|
|
| 10 |
import time
|
| 11 |
|
| 12 |
import requests
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 16 |
+
from src.services.brain_identity import InvalidBrainIdError, validate_brain_id
|
| 17 |
|
| 18 |
|
| 19 |
DEFAULT_DOCS_URL = "https://brainapi.lumen-labs.ai/docs/llms-full.txt"
|
|
|
|
| 31 |
if not token:
|
| 32 |
print("BRAINPAT_TOKEN is required", file=sys.stderr)
|
| 33 |
return 2
|
| 34 |
+
try:
|
| 35 |
+
args.brain = validate_brain_id(args.brain)
|
| 36 |
+
except InvalidBrainIdError as error:
|
| 37 |
+
print(str(error), file=sys.stderr)
|
| 38 |
return 2
|
| 39 |
|
| 40 |
api_url = args.api_url.rstrip("/")
|
|
|
|
| 54 |
if existing.status_code == 200 and existing.json().get("total", 0) > 0:
|
| 55 |
print(f"Public demo brain {args.brain!r} is already current ({digest[:12]}).")
|
| 56 |
return 0
|
| 57 |
+
if existing.status_code in (404, 406) and existing.json().get("error", {}).get("code") == "BRAIN_NOT_FOUND":
|
| 58 |
+
created = requests.post(
|
| 59 |
+
f"{api_url}/system/brains", headers=headers,
|
| 60 |
+
json={"brain_id": args.brain}, timeout=30,
|
| 61 |
+
)
|
| 62 |
+
if not (created.status_code == 409 and created.json().get("error", {}).get("code") == "BRAIN_ALREADY_EXISTS"):
|
| 63 |
+
created.raise_for_status()
|
| 64 |
+
else:
|
| 65 |
existing.raise_for_status()
|
| 66 |
|
| 67 |
payload = {
|
src/adapters/data.py
CHANGED
|
@@ -99,7 +99,17 @@ class DataAdapter:
|
|
| 99 |
"""
|
| 100 |
Create a new brain in the data client.
|
| 101 |
"""
|
| 102 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
def get_brain(self, name_key: str) -> Brain:
|
| 105 |
"""
|
|
|
|
| 99 |
"""
|
| 100 |
Create a new brain in the data client.
|
| 101 |
"""
|
| 102 |
+
brain = self.data.create_brain(name_key=name_key)
|
| 103 |
+
from src.config import config
|
| 104 |
+
if config.data_db.strip().lower() != "postgresql" and (
|
| 105 |
+
config.vector_db.strip().lower() == "postgresql"
|
| 106 |
+
or config.graph_db.strip().lower() == "networkx"
|
| 107 |
+
):
|
| 108 |
+
# A Mongo registry with PostgreSQL graph/vectors also provisions only
|
| 109 |
+
# at this explicit creation boundary, never from backend reads.
|
| 110 |
+
from src.lib.postgresql._provisioning import ensure_brain_database
|
| 111 |
+
ensure_brain_database(brain.name_key)
|
| 112 |
+
return brain
|
| 113 |
|
| 114 |
def get_brain(self, name_key: str) -> Brain:
|
| 115 |
"""
|
src/config.py
CHANGED
|
@@ -389,16 +389,8 @@ class EmbeddingsConfig:
|
|
| 389 |
self.local_model = os.getenv("EMBEDDINGS_LOCAL_MODEL", "BAAI/bge-small-en-v1.5")
|
| 390 |
self.small_model = os.getenv("EMBEDDINGS_SMALL_MODEL", "BAAI/bge-small-en-v1.5")
|
| 391 |
|
| 392 |
-
provider = (provider or "").strip().lower()
|
| 393 |
-
model = (model or "").strip()
|
| 394 |
gemma = provider == "ollama" and model == "embeddinggemma-2:270m"
|
| 395 |
-
self.profile = (
|
| 396 |
-
os.getenv(
|
| 397 |
-
"EMBEDDING_PROFILE",
|
| 398 |
-
"embeddinggemma2-text-v1" if gemma else "",
|
| 399 |
-
)
|
| 400 |
-
or ""
|
| 401 |
-
).strip()
|
| 402 |
default_dimension = 512 if gemma else 384
|
| 403 |
|
| 404 |
self.embedding_nodes_dimension = (
|
|
@@ -438,17 +430,7 @@ class EmbeddingsConfig:
|
|
| 438 |
if gemma or self.profile == "embeddinggemma2-text-v1":
|
| 439 |
if not (gemma and self.profile == "embeddinggemma2-text-v1" and self.native_dimension == 768
|
| 440 |
and self.mrl_dimension == 512 and dimensions == {512}):
|
| 441 |
-
raise ValueError(
|
| 442 |
-
"embeddinggemma2-text-v1 configuration mismatch: "
|
| 443 |
-
f"provider={provider!r}, model={model!r}, "
|
| 444 |
-
f"profile={self.profile!r}, "
|
| 445 |
-
f"native_dimension={self.native_dimension!r}, "
|
| 446 |
-
f"mrl_dimension={self.mrl_dimension!r}, "
|
| 447 |
-
f"stored_dimensions={sorted(dimensions)!r}; "
|
| 448 |
-
"expected provider='ollama', model='embeddinggemma-2:270m', "
|
| 449 |
-
"profile='embeddinggemma2-text-v1', native=768, MRL=512, "
|
| 450 |
-
"all stored dimensions=512"
|
| 451 |
-
)
|
| 452 |
elif self.mrl_dimension:
|
| 453 |
raise ValueError("MRL is only supported by the qualified embeddinggemma2-text-v1 profile")
|
| 454 |
|
|
|
|
| 389 |
self.local_model = os.getenv("EMBEDDINGS_LOCAL_MODEL", "BAAI/bge-small-en-v1.5")
|
| 390 |
self.small_model = os.getenv("EMBEDDINGS_SMALL_MODEL", "BAAI/bge-small-en-v1.5")
|
| 391 |
|
|
|
|
|
|
|
| 392 |
gemma = provider == "ollama" and model == "embeddinggemma-2:270m"
|
| 393 |
+
self.profile = os.getenv("EMBEDDING_PROFILE", "embeddinggemma2-text-v1" if gemma else "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
default_dimension = 512 if gemma else 384
|
| 395 |
|
| 396 |
self.embedding_nodes_dimension = (
|
|
|
|
| 430 |
if gemma or self.profile == "embeddinggemma2-text-v1":
|
| 431 |
if not (gemma and self.profile == "embeddinggemma2-text-v1" and self.native_dimension == 768
|
| 432 |
and self.mrl_dimension == 512 and dimensions == {512}):
|
| 433 |
+
raise ValueError("embeddinggemma2-text-v1 requires ollama embeddinggemma-2:270m, native 768, MRL 512 and all stored dimensions 512; use a new index when changing profiles")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 434 |
elif self.mrl_dimension:
|
| 435 |
raise ValueError("MRL is only supported by the qualified embeddinggemma2-text-v1 profile")
|
| 436 |
|
src/lib/mongo/client.py
CHANGED
|
@@ -16,6 +16,7 @@ from pymongo.database import Database
|
|
| 16 |
from pymongo.collection import Collection
|
| 17 |
from src.adapters.interfaces.data import DataClient, SearchResult
|
| 18 |
from src.config import config
|
|
|
|
| 19 |
from src.constants.data import Brain, KGChanges, Observation, StructuredData, TextChunk, Workspace
|
| 20 |
|
| 21 |
|
|
@@ -139,10 +140,15 @@ class MongoClient(DataClient):
|
|
| 139 |
return structured_data
|
| 140 |
|
| 141 |
def create_brain(self, name_key: str) -> Brain:
|
|
|
|
| 142 |
collection = self.get_collection("brains", config.mongo.system_database)
|
| 143 |
brain = Brain(name_key=name_key)
|
| 144 |
brain_dict = brain.model_dump(mode="json", exclude={"id"})
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
brain.id = str(result.inserted_id)
|
| 147 |
return brain
|
| 148 |
|
|
|
|
| 16 |
from pymongo.collection import Collection
|
| 17 |
from src.adapters.interfaces.data import DataClient, SearchResult
|
| 18 |
from src.config import config
|
| 19 |
+
from src.services.brain_identity import BrainAlreadyExistsError, validate_brain_id
|
| 20 |
from src.constants.data import Brain, KGChanges, Observation, StructuredData, TextChunk, Workspace
|
| 21 |
|
| 22 |
|
|
|
|
| 140 |
return structured_data
|
| 141 |
|
| 142 |
def create_brain(self, name_key: str) -> Brain:
|
| 143 |
+
name_key = validate_brain_id(name_key)
|
| 144 |
collection = self.get_collection("brains", config.mongo.system_database)
|
| 145 |
brain = Brain(name_key=name_key)
|
| 146 |
brain_dict = brain.model_dump(mode="json", exclude={"id"})
|
| 147 |
+
collection.create_index("name_key", unique=True)
|
| 148 |
+
try:
|
| 149 |
+
result = collection.insert_one(brain_dict)
|
| 150 |
+
except DuplicateKeyError:
|
| 151 |
+
raise BrainAlreadyExistsError("Brain already exists") from None
|
| 152 |
brain.id = str(result.inserted_id)
|
| 153 |
return brain
|
| 154 |
|
src/lib/postgresql/_provisioning.py
CHANGED
|
@@ -35,6 +35,7 @@ import psycopg2.pool
|
|
| 35 |
from psycopg2 import sql
|
| 36 |
|
| 37 |
from src.config import config
|
|
|
|
| 38 |
|
| 39 |
from ._naming import BRAIN_DB_PREFIX, brain_db_name
|
| 40 |
|
|
@@ -101,6 +102,12 @@ class _PoolRegistry:
|
|
| 101 |
self._evict_if_needed()
|
| 102 |
return pool
|
| 103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
def _evict_if_needed(self) -> None:
|
| 105 |
while len(self._pools) > _MAX_OPEN_POOLS:
|
| 106 |
evicted_name, evicted_pool = self._pools.popitem(last=False)
|
|
@@ -124,14 +131,34 @@ def get_system_pool() -> psycopg2.pool.ThreadedConnectionPool:
|
|
| 124 |
|
| 125 |
|
| 126 |
def get_brain_pool(brain_id: str) -> psycopg2.pool.ThreadedConnectionPool:
|
|
|
|
| 127 |
return _registry.get(brain_db_name(brain_id))
|
| 128 |
|
| 129 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
@contextmanager
|
| 131 |
def borrow(pool: psycopg2.pool.ThreadedConnectionPool) -> Iterator[psycopg2.extensions.connection]:
|
| 132 |
conn = pool.getconn()
|
| 133 |
try:
|
| 134 |
yield conn
|
|
|
|
|
|
|
|
|
|
| 135 |
finally:
|
| 136 |
pool.putconn(conn)
|
| 137 |
|
|
@@ -181,6 +208,7 @@ def ensure_database_exists(dbname: str) -> None:
|
|
| 181 |
|
| 182 |
def ensure_brain_database(brain_id: str) -> str:
|
| 183 |
"""Return the per-brain database name, creating the database if needed."""
|
|
|
|
| 184 |
dbname = brain_db_name(brain_id)
|
| 185 |
ensure_database_exists(dbname)
|
| 186 |
return dbname
|
|
@@ -232,7 +260,7 @@ def clear_brain_database(
|
|
| 232 |
if cur.fetchone() is None:
|
| 233 |
return []
|
| 234 |
|
| 235 |
-
with borrow(
|
| 236 |
with conn.cursor() as cur:
|
| 237 |
cur.execute(
|
| 238 |
"SELECT tablename FROM pg_tables "
|
|
@@ -255,3 +283,24 @@ def clear_brain_database(
|
|
| 255 |
)
|
| 256 |
conn.commit()
|
| 257 |
return tables
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
from psycopg2 import sql
|
| 36 |
|
| 37 |
from src.config import config
|
| 38 |
+
from src.services.brain_identity import BrainNotFoundError, validate_brain_id
|
| 39 |
|
| 40 |
from ._naming import BRAIN_DB_PREFIX, brain_db_name
|
| 41 |
|
|
|
|
| 102 |
self._evict_if_needed()
|
| 103 |
return pool
|
| 104 |
|
| 105 |
+
def discard(self, dbname: str) -> None:
|
| 106 |
+
with self._lock:
|
| 107 |
+
pool = self._pools.pop(dbname, None)
|
| 108 |
+
if pool is not None:
|
| 109 |
+
pool.closeall()
|
| 110 |
+
|
| 111 |
def _evict_if_needed(self) -> None:
|
| 112 |
while len(self._pools) > _MAX_OPEN_POOLS:
|
| 113 |
evicted_name, evicted_pool = self._pools.popitem(last=False)
|
|
|
|
| 131 |
|
| 132 |
|
| 133 |
def get_brain_pool(brain_id: str) -> psycopg2.pool.ThreadedConnectionPool:
|
| 134 |
+
require_registered_brain(brain_id)
|
| 135 |
return _registry.get(brain_db_name(brain_id))
|
| 136 |
|
| 137 |
|
| 138 |
+
def require_registered_brain(brain_id: str) -> None:
|
| 139 |
+
"""Authorize an existing namespace from persisted state, never by storage/cache."""
|
| 140 |
+
brain_id = validate_brain_id(brain_id)
|
| 141 |
+
if config.data_db.strip().lower() != "postgresql":
|
| 142 |
+
# Mixed storage profiles still use their configured canonical registry.
|
| 143 |
+
from src.services.data.main import data_adapter
|
| 144 |
+
if data_adapter.get_brain(name_key=brain_id) is None:
|
| 145 |
+
raise BrainNotFoundError("Brain not found")
|
| 146 |
+
return
|
| 147 |
+
with borrow(get_system_pool()) as conn:
|
| 148 |
+
with conn.cursor() as cur:
|
| 149 |
+
cur.execute("SELECT 1 FROM data_brains WHERE name_key = %s", (brain_id,))
|
| 150 |
+
if cur.fetchone() is None:
|
| 151 |
+
raise BrainNotFoundError("Brain not found")
|
| 152 |
+
|
| 153 |
+
|
| 154 |
@contextmanager
|
| 155 |
def borrow(pool: psycopg2.pool.ThreadedConnectionPool) -> Iterator[psycopg2.extensions.connection]:
|
| 156 |
conn = pool.getconn()
|
| 157 |
try:
|
| 158 |
yield conn
|
| 159 |
+
except BaseException:
|
| 160 |
+
conn.rollback()
|
| 161 |
+
raise
|
| 162 |
finally:
|
| 163 |
pool.putconn(conn)
|
| 164 |
|
|
|
|
| 208 |
|
| 209 |
def ensure_brain_database(brain_id: str) -> str:
|
| 210 |
"""Return the per-brain database name, creating the database if needed."""
|
| 211 |
+
require_registered_brain(brain_id)
|
| 212 |
dbname = brain_db_name(brain_id)
|
| 213 |
ensure_database_exists(dbname)
|
| 214 |
return dbname
|
|
|
|
| 260 |
if cur.fetchone() is None:
|
| 261 |
return []
|
| 262 |
|
| 263 |
+
with borrow(get_pool(dbname)) as conn:
|
| 264 |
with conn.cursor() as cur:
|
| 265 |
cur.execute(
|
| 266 |
"SELECT tablename FROM pg_tables "
|
|
|
|
| 283 |
)
|
| 284 |
conn.commit()
|
| 285 |
return tables
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def drop_brain_database(brain_id: str) -> None:
|
| 289 |
+
"""Explicit lifecycle delete, including unregistered orphan storage."""
|
| 290 |
+
brain_id = validate_brain_id(brain_id)
|
| 291 |
+
dbname = brain_db_name(brain_id)
|
| 292 |
+
if dbname in {_system_dbname(), _maintenance_dbname()}:
|
| 293 |
+
raise ValueError("Cannot delete an infrastructure database")
|
| 294 |
+
_registry.discard(dbname)
|
| 295 |
+
conn = psycopg2.connect(
|
| 296 |
+
host=config.postgresql.host, port=config.postgresql.port,
|
| 297 |
+
user=config.postgresql.username, password=config.postgresql.password,
|
| 298 |
+
dbname=_maintenance_dbname(),
|
| 299 |
+
)
|
| 300 |
+
try:
|
| 301 |
+
conn.autocommit = True
|
| 302 |
+
with conn.cursor() as cur:
|
| 303 |
+
cur.execute(sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(sql.Identifier(dbname)))
|
| 304 |
+
finally:
|
| 305 |
+
conn.close()
|
| 306 |
+
_created_dbs.discard(dbname)
|
src/lib/postgresql/data.py
CHANGED
|
@@ -17,6 +17,7 @@ import psycopg2.extras
|
|
| 17 |
|
| 18 |
from src.adapters.interfaces.data import DataClient, SearchResult
|
| 19 |
from src.config import SEARCH_FTS_REGCONFIGS, config
|
|
|
|
| 20 |
from src.constants.data import (
|
| 21 |
Brain,
|
| 22 |
KGChanges,
|
|
@@ -27,11 +28,6 @@ from src.constants.data import (
|
|
| 27 |
Workspace,
|
| 28 |
)
|
| 29 |
|
| 30 |
-
from ._naming import (
|
| 31 |
-
brain_db_name,
|
| 32 |
-
brain_id_from_db_name,
|
| 33 |
-
is_internal_brain_db_suffix,
|
| 34 |
-
)
|
| 35 |
from ._provisioning import (
|
| 36 |
borrow,
|
| 37 |
ensure_brain_database,
|
|
@@ -39,7 +35,8 @@ from ._provisioning import (
|
|
| 39 |
get_brain_pool,
|
| 40 |
get_system_pool,
|
| 41 |
clear_brain_database,
|
| 42 |
-
|
|
|
|
| 43 |
)
|
| 44 |
|
| 45 |
|
|
@@ -272,7 +269,7 @@ class PostgreSQLDataClient(DataClient):
|
|
| 272 |
def _brain_connection(
|
| 273 |
self, brain_id: str
|
| 274 |
) -> Iterator[psycopg2.extensions.connection]:
|
| 275 |
-
|
| 276 |
with borrow(get_brain_pool(brain_id)) as conn:
|
| 277 |
yield conn
|
| 278 |
|
|
@@ -290,6 +287,7 @@ class PostgreSQLDataClient(DataClient):
|
|
| 290 |
self._system_initialized = True
|
| 291 |
|
| 292 |
def _ensure_brain_schema(self, brain_id: str) -> None:
|
|
|
|
| 293 |
alt_regconfig = config.search_fts_regconfig_for_brain(brain_id)
|
| 294 |
needs_base = brain_id not in self._initialized_brains
|
| 295 |
needs_search = (
|
|
@@ -536,23 +534,27 @@ class PostgreSQLDataClient(DataClient):
|
|
| 536 |
return structured_data
|
| 537 |
|
| 538 |
def create_brain(self, name_key: str) -> Brain:
|
|
|
|
| 539 |
brain = Brain(name_key=name_key)
|
| 540 |
document = brain.model_dump(mode="json")
|
| 541 |
-
|
| 542 |
-
with
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
|
|
|
|
|
|
|
|
|
| 556 |
self._ensure_brain_schema(name_key)
|
| 557 |
return brain
|
| 558 |
|
|
@@ -585,22 +587,10 @@ class PostgreSQLDataClient(DataClient):
|
|
| 585 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 586 |
cur.execute("SELECT document FROM data_brains")
|
| 587 |
rows = cur.fetchall()
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
} | {brain_db_name(brain.id) for brain in registered}
|
| 593 |
-
|
| 594 |
-
for db_name in list_brain_database_names():
|
| 595 |
-
if db_name in known_db_names:
|
| 596 |
-
continue
|
| 597 |
-
suffix = brain_id_from_db_name(db_name)
|
| 598 |
-
if not suffix or is_internal_brain_db_suffix(suffix):
|
| 599 |
-
continue
|
| 600 |
-
if suffix not in by_key:
|
| 601 |
-
by_key[suffix] = Brain(name_key=suffix)
|
| 602 |
-
|
| 603 |
-
return sorted(by_key.values(), key=lambda brain: brain.name_key)
|
| 604 |
|
| 605 |
def create_workspace(self, workspace: Workspace) -> Workspace:
|
| 606 |
with self._system_connection() as conn:
|
|
@@ -647,6 +637,12 @@ class PostgreSQLDataClient(DataClient):
|
|
| 647 |
def clear_brain_data(self, brain_id: str) -> None:
|
| 648 |
clear_brain_database(brain_id)
|
| 649 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 650 |
def delete_brain_registry(self, brain_id: str) -> bool:
|
| 651 |
with self._system_connection() as conn:
|
| 652 |
with conn.cursor() as cur:
|
|
|
|
| 17 |
|
| 18 |
from src.adapters.interfaces.data import DataClient, SearchResult
|
| 19 |
from src.config import SEARCH_FTS_REGCONFIGS, config
|
| 20 |
+
from src.services.brain_identity import BrainAlreadyExistsError, validate_brain_id
|
| 21 |
from src.constants.data import (
|
| 22 |
Brain,
|
| 23 |
KGChanges,
|
|
|
|
| 28 |
Workspace,
|
| 29 |
)
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
from ._provisioning import (
|
| 32 |
borrow,
|
| 33 |
ensure_brain_database,
|
|
|
|
| 35 |
get_brain_pool,
|
| 36 |
get_system_pool,
|
| 37 |
clear_brain_database,
|
| 38 |
+
drop_brain_database,
|
| 39 |
+
require_registered_brain,
|
| 40 |
)
|
| 41 |
|
| 42 |
|
|
|
|
| 269 |
def _brain_connection(
|
| 270 |
self, brain_id: str
|
| 271 |
) -> Iterator[psycopg2.extensions.connection]:
|
| 272 |
+
# Existing connection only: provisioning belongs to create_brain().
|
| 273 |
with borrow(get_brain_pool(brain_id)) as conn:
|
| 274 |
yield conn
|
| 275 |
|
|
|
|
| 287 |
self._system_initialized = True
|
| 288 |
|
| 289 |
def _ensure_brain_schema(self, brain_id: str) -> None:
|
| 290 |
+
require_registered_brain(brain_id)
|
| 291 |
alt_regconfig = config.search_fts_regconfig_for_brain(brain_id)
|
| 292 |
needs_base = brain_id not in self._initialized_brains
|
| 293 |
needs_search = (
|
|
|
|
| 534 |
return structured_data
|
| 535 |
|
| 536 |
def create_brain(self, name_key: str) -> Brain:
|
| 537 |
+
name_key = validate_brain_id(name_key)
|
| 538 |
brain = Brain(name_key=name_key)
|
| 539 |
document = brain.model_dump(mode="json")
|
| 540 |
+
try:
|
| 541 |
+
with self._system_connection() as conn:
|
| 542 |
+
with conn.cursor() as cur:
|
| 543 |
+
cur.execute(
|
| 544 |
+
"""
|
| 545 |
+
INSERT INTO data_brains (id, name_key, pat, document)
|
| 546 |
+
VALUES (%s, %s, %s, %s::jsonb)
|
| 547 |
+
""",
|
| 548 |
+
(
|
| 549 |
+
brain.id,
|
| 550 |
+
brain.name_key,
|
| 551 |
+
brain.pat,
|
| 552 |
+
json.dumps(document, default=str),
|
| 553 |
+
),
|
| 554 |
+
)
|
| 555 |
+
conn.commit()
|
| 556 |
+
except psycopg2.errors.UniqueViolation:
|
| 557 |
+
raise BrainAlreadyExistsError("Brain already exists") from None
|
| 558 |
self._ensure_brain_schema(name_key)
|
| 559 |
return brain
|
| 560 |
|
|
|
|
| 587 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 588 |
cur.execute("SELECT document FROM data_brains")
|
| 589 |
rows = cur.fetchall()
|
| 590 |
+
return sorted(
|
| 591 |
+
(Brain.model_validate(row["document"]) for row in rows),
|
| 592 |
+
key=lambda brain: brain.name_key,
|
| 593 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 594 |
|
| 595 |
def create_workspace(self, workspace: Workspace) -> Workspace:
|
| 596 |
with self._system_connection() as conn:
|
|
|
|
| 637 |
def clear_brain_data(self, brain_id: str) -> None:
|
| 638 |
clear_brain_database(brain_id)
|
| 639 |
|
| 640 |
+
def delete_brain_storage(self, brain_id: str) -> None:
|
| 641 |
+
drop_brain_database(brain_id)
|
| 642 |
+
self._initialized_brains.discard(brain_id)
|
| 643 |
+
self._search_ready_brains.discard(brain_id)
|
| 644 |
+
self._search_alt_brains.discard(brain_id)
|
| 645 |
+
|
| 646 |
def delete_brain_registry(self, brain_id: str) -> bool:
|
| 647 |
with self._system_connection() as conn:
|
| 648 |
with conn.cursor() as cur:
|
src/lib/postgresql/graph_store.py
CHANGED
|
@@ -19,7 +19,7 @@ import psycopg2.extras
|
|
| 19 |
|
| 20 |
from src.config import config
|
| 21 |
|
| 22 |
-
from ._provisioning import borrow,
|
| 23 |
from .read_query import (
|
| 24 |
MAX_READ_QUERY_ROWS,
|
| 25 |
READ_QUERY_TIMEOUT_MS,
|
|
@@ -152,9 +152,9 @@ class PostgreSQLGraphStore:
|
|
| 152 |
"""
|
| 153 |
Graph driver that persists each brain into its own Postgres database.
|
| 154 |
|
| 155 |
-
The per-brain database is provisioned
|
| 156 |
-
|
| 157 |
-
schema (no ``brain_id`` columns, no ``kg_brains`` reference table) because
|
| 158 |
the brain identity is encoded in the database name itself.
|
| 159 |
"""
|
| 160 |
|
|
@@ -226,6 +226,7 @@ class PostgreSQLGraphStore:
|
|
| 226 |
self._brains_lock = threading.RLock()
|
| 227 |
|
| 228 |
def _ensure_brain_schema(self, brain_id: str) -> None:
|
|
|
|
| 229 |
needs_base = brain_id not in self._schema_ready
|
| 230 |
needs_search = (
|
| 231 |
bool(config.search_enabled) and brain_id not in self._search_ready_brains
|
|
@@ -240,7 +241,6 @@ class PostgreSQLGraphStore:
|
|
| 240 |
)
|
| 241 |
if not needs_base and not needs_search:
|
| 242 |
return
|
| 243 |
-
ensure_brain_database(brain_id)
|
| 244 |
with borrow(get_brain_pool(brain_id)) as conn:
|
| 245 |
if needs_base:
|
| 246 |
with conn.cursor() as cur:
|
|
@@ -384,6 +384,8 @@ class PostgreSQLGraphStore:
|
|
| 384 |
def invalidate_brain(self, brain_id: str) -> None:
|
| 385 |
with self._brains_lock:
|
| 386 |
self._brains.pop(brain_id, None)
|
|
|
|
|
|
|
| 387 |
|
| 388 |
def _db_graph_counts(self, brain_id: str) -> tuple[int, int]:
|
| 389 |
self._ensure_brain_schema(brain_id)
|
|
@@ -397,6 +399,7 @@ class PostgreSQLGraphStore:
|
|
| 397 |
|
| 398 |
def _load_brain(self, brain_id: str) -> _BrainGraph:
|
| 399 |
"""Read path: refresh from Postgres when another process wrote data."""
|
|
|
|
| 400 |
with self._brains_lock:
|
| 401 |
cached = self._brains.get(brain_id)
|
| 402 |
cached_counts = (
|
|
|
|
| 19 |
|
| 20 |
from src.config import config
|
| 21 |
|
| 22 |
+
from ._provisioning import borrow, require_registered_brain, get_brain_pool
|
| 23 |
from .read_query import (
|
| 24 |
MAX_READ_QUERY_ROWS,
|
| 25 |
READ_QUERY_TIMEOUT_MS,
|
|
|
|
| 152 |
"""
|
| 153 |
Graph driver that persists each brain into its own Postgres database.
|
| 154 |
|
| 155 |
+
The per-brain database is provisioned explicitly at brain creation, then
|
| 156 |
+
accessed through registry-checked connections in the shared LRU pool registry.
|
| 157 |
+
Each database owns a flat schema (no ``brain_id`` columns, no ``kg_brains`` reference table) because
|
| 158 |
the brain identity is encoded in the database name itself.
|
| 159 |
"""
|
| 160 |
|
|
|
|
| 226 |
self._brains_lock = threading.RLock()
|
| 227 |
|
| 228 |
def _ensure_brain_schema(self, brain_id: str) -> None:
|
| 229 |
+
require_registered_brain(brain_id)
|
| 230 |
needs_base = brain_id not in self._schema_ready
|
| 231 |
needs_search = (
|
| 232 |
bool(config.search_enabled) and brain_id not in self._search_ready_brains
|
|
|
|
| 241 |
)
|
| 242 |
if not needs_base and not needs_search:
|
| 243 |
return
|
|
|
|
| 244 |
with borrow(get_brain_pool(brain_id)) as conn:
|
| 245 |
if needs_base:
|
| 246 |
with conn.cursor() as cur:
|
|
|
|
| 384 |
def invalidate_brain(self, brain_id: str) -> None:
|
| 385 |
with self._brains_lock:
|
| 386 |
self._brains.pop(brain_id, None)
|
| 387 |
+
self._schema_ready.discard(brain_id)
|
| 388 |
+
self._search_ready_brains.discard(brain_id)
|
| 389 |
|
| 390 |
def _db_graph_counts(self, brain_id: str) -> tuple[int, int]:
|
| 391 |
self._ensure_brain_schema(brain_id)
|
|
|
|
| 399 |
|
| 400 |
def _load_brain(self, brain_id: str) -> _BrainGraph:
|
| 401 |
"""Read path: refresh from Postgres when another process wrote data."""
|
| 402 |
+
require_registered_brain(brain_id)
|
| 403 |
with self._brains_lock:
|
| 404 |
cached = self._brains.get(brain_id)
|
| 405 |
cached_counts = (
|
src/lib/postgresql/vectors.py
CHANGED
|
@@ -27,7 +27,7 @@ from src.lib.embeddings.profile import EmbeddingProfileError, index_profile, req
|
|
| 27 |
from ._provisioning import (
|
| 28 |
borrow,
|
| 29 |
clear_brain_database,
|
| 30 |
-
|
| 31 |
get_brain_pool,
|
| 32 |
)
|
| 33 |
|
|
@@ -136,12 +136,12 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):
|
|
| 136 |
yield conn
|
| 137 |
|
| 138 |
def _ensure_extension(self, brain_id: str) -> None:
|
|
|
|
| 139 |
if brain_id in self._brain_extensions_ready:
|
| 140 |
return
|
| 141 |
with self._lock:
|
| 142 |
if brain_id in self._brain_extensions_ready:
|
| 143 |
return
|
| 144 |
-
ensure_brain_database(brain_id)
|
| 145 |
with borrow(get_brain_pool(brain_id)) as conn:
|
| 146 |
with conn.cursor() as cur:
|
| 147 |
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
@@ -150,8 +150,13 @@ class PostgreSQLVectorStoreClient(VectorStoreClient):
|
|
| 150 |
|
| 151 |
def clear_brain_data(self, brain_id: str) -> None:
|
| 152 |
clear_brain_database(brain_id, table_prefixes=("vectors_",))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
def _ensure_store(self, store: str, brain_id: str) -> None:
|
|
|
|
| 155 |
if store not in EMBEDDING_STORES_SIZES:
|
| 156 |
raise ValueError(f"Store {store} not available")
|
| 157 |
dimension = EMBEDDING_STORES_SIZES[store]
|
|
|
|
| 27 |
from ._provisioning import (
|
| 28 |
borrow,
|
| 29 |
clear_brain_database,
|
| 30 |
+
require_registered_brain,
|
| 31 |
get_brain_pool,
|
| 32 |
)
|
| 33 |
|
|
|
|
| 136 |
yield conn
|
| 137 |
|
| 138 |
def _ensure_extension(self, brain_id: str) -> None:
|
| 139 |
+
require_registered_brain(brain_id)
|
| 140 |
if brain_id in self._brain_extensions_ready:
|
| 141 |
return
|
| 142 |
with self._lock:
|
| 143 |
if brain_id in self._brain_extensions_ready:
|
| 144 |
return
|
|
|
|
| 145 |
with borrow(get_brain_pool(brain_id)) as conn:
|
| 146 |
with conn.cursor() as cur:
|
| 147 |
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
|
|
| 150 |
|
| 151 |
def clear_brain_data(self, brain_id: str) -> None:
|
| 152 |
clear_brain_database(brain_id, table_prefixes=("vectors_",))
|
| 153 |
+
self._brain_extensions_ready.discard(brain_id)
|
| 154 |
+
self._initialized_stores = {
|
| 155 |
+
key for key in self._initialized_stores if not key.startswith(f"{brain_id}:")
|
| 156 |
+
}
|
| 157 |
|
| 158 |
def _ensure_store(self, store: str, brain_id: str) -> None:
|
| 159 |
+
require_registered_brain(brain_id)
|
| 160 |
if store not in EMBEDDING_STORES_SIZES:
|
| 161 |
raise ValueError(f"Store {store} not available")
|
| 162 |
dimension = EMBEDDING_STORES_SIZES[store]
|
src/services/api/controllers/system.py
CHANGED
|
@@ -9,7 +9,7 @@ Modified By: Christian Nonis <alch.infoemail@gmail.com>
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
import asyncio
|
| 12 |
-
import
|
| 13 |
|
| 14 |
from src.services.api.constants.requests import CreateBrainRequest
|
| 15 |
from src.services.brain_lifecycle import BrainLifecycleService
|
|
@@ -28,9 +28,8 @@ async def create_new_brain(request: CreateBrainRequest):
|
|
| 28 |
"""
|
| 29 |
Create a new brain
|
| 30 |
"""
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
result = await asyncio.to_thread(data_adapter.create_brain, request.brain_id)
|
| 34 |
return result
|
| 35 |
|
| 36 |
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
import asyncio
|
| 12 |
+
from src.services.brain_identity import validate_brain_id
|
| 13 |
|
| 14 |
from src.services.api.constants.requests import CreateBrainRequest
|
| 15 |
from src.services.brain_lifecycle import BrainLifecycleService
|
|
|
|
| 28 |
"""
|
| 29 |
Create a new brain
|
| 30 |
"""
|
| 31 |
+
brain_id = validate_brain_id(request.brain_id)
|
| 32 |
+
result = await asyncio.to_thread(data_adapter.create_brain, brain_id)
|
|
|
|
| 33 |
return result
|
| 34 |
|
| 35 |
|
src/services/api/errors.py
CHANGED
|
@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse
|
|
| 12 |
from pydantic import BaseModel, Field, JsonValue
|
| 13 |
from starlette.exceptions import HTTPException as StarletteHTTPException
|
| 14 |
from src.lib.embeddings.profile import EmbeddingProfileError
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
class ErrorInfo(BaseModel):
|
|
@@ -158,6 +159,25 @@ def install_error_handlers(app: FastAPI) -> None:
|
|
| 158 |
detail=exc.errors(),
|
| 159 |
)
|
| 160 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
@app.exception_handler(EmbeddingProfileError)
|
| 162 |
async def embedding_profile_exception_handler(request: Request, exc: EmbeddingProfileError):
|
| 163 |
return error_response(
|
|
|
|
| 12 |
from pydantic import BaseModel, Field, JsonValue
|
| 13 |
from starlette.exceptions import HTTPException as StarletteHTTPException
|
| 14 |
from src.lib.embeddings.profile import EmbeddingProfileError
|
| 15 |
+
from src.services.brain_identity import BrainNotFoundError, BrainAlreadyExistsError, InvalidBrainIdError
|
| 16 |
|
| 17 |
|
| 18 |
class ErrorInfo(BaseModel):
|
|
|
|
| 159 |
detail=exc.errors(),
|
| 160 |
)
|
| 161 |
|
| 162 |
+
@app.exception_handler(BrainNotFoundError)
|
| 163 |
+
async def missing_brain_handler(request: Request, _exc: BrainNotFoundError):
|
| 164 |
+
return error_response(
|
| 165 |
+
request, status_code=404, detail="Brain not found", code="BRAIN_NOT_FOUND",
|
| 166 |
+
resolution="Create the brain explicitly or select a registered brain.",
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
@app.exception_handler(BrainAlreadyExistsError)
|
| 170 |
+
async def duplicate_brain_handler(request: Request, _exc: BrainAlreadyExistsError):
|
| 171 |
+
return error_response(
|
| 172 |
+
request, status_code=409, detail="Brain already exists", code="BRAIN_ALREADY_EXISTS",
|
| 173 |
+
message="A brain with this identifier already exists.",
|
| 174 |
+
resolution="Select the existing brain or create a different identifier.",
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
@app.exception_handler(InvalidBrainIdError)
|
| 178 |
+
async def invalid_brain_handler(request: Request, exc: InvalidBrainIdError):
|
| 179 |
+
return error_response(request, status_code=400, detail=str(exc), code=exc.code)
|
| 180 |
+
|
| 181 |
@app.exception_handler(EmbeddingProfileError)
|
| 182 |
async def embedding_profile_exception_handler(request: Request, exc: EmbeddingProfileError):
|
| 183 |
return error_response(
|
src/services/api/middlewares/auth.py
CHANGED
|
@@ -94,7 +94,7 @@ class BrainPATMiddleware(BaseHTTPMiddleware):
|
|
| 94 |
detail="Brain ID is required.",
|
| 95 |
code="BRAIN_ID_REQUIRED",
|
| 96 |
message="A brain identifier is required.",
|
| 97 |
-
resolution="Send X-Brain-ID
|
| 98 |
)
|
| 99 |
cachepat_key = f"brainpat:{brain_id}"
|
| 100 |
if system_pat and secrets.compare_digest(brainpat, system_pat):
|
|
|
|
| 94 |
detail="Brain ID is required.",
|
| 95 |
code="BRAIN_ID_REQUIRED",
|
| 96 |
message="A brain identifier is required.",
|
| 97 |
+
resolution="Send X-Brain-ID using 1-128 ASCII letters, digits, hyphens, or underscores, starting with a letter or digit.",
|
| 98 |
)
|
| 99 |
cachepat_key = f"brainpat:{brain_id}"
|
| 100 |
if system_pat and secrets.compare_digest(brainpat, system_pat):
|
src/services/api/middlewares/brains.py
CHANGED
|
@@ -16,12 +16,11 @@ from email.parser import BytesParser
|
|
| 16 |
|
| 17 |
from fastapi import Request
|
| 18 |
from starlette.middleware.base import BaseHTTPMiddleware
|
| 19 |
-
from starlette import status
|
| 20 |
|
| 21 |
from src.services.api.console_static import is_console_path
|
| 22 |
from src.services.api.errors import error_response
|
| 23 |
from src.services.data.main import data_adapter
|
| 24 |
-
from src.services.
|
| 25 |
from src.services.workspaces import WorkspaceScopeConflictError, resolve_workspace_scope
|
| 26 |
|
| 27 |
|
|
@@ -102,7 +101,7 @@ class BrainMiddleware(BaseHTTPMiddleware):
|
|
| 102 |
body_data = json.loads(body)
|
| 103 |
if isinstance(body_data, dict):
|
| 104 |
body_brain_id = body_data.get("brain_id")
|
| 105 |
-
if body_brain_id:
|
| 106 |
body_brain_id = body_brain_id.rstrip()
|
| 107 |
except (json.JSONDecodeError, ValueError):
|
| 108 |
pass
|
|
@@ -163,83 +162,41 @@ class BrainMiddleware(BaseHTTPMiddleware):
|
|
| 163 |
brain_id, scope_error = await _get_brain_id()
|
| 164 |
if scope_error is not None:
|
| 165 |
return scope_error
|
| 166 |
-
brain_creation_allowed = os.getenv("BRAIN_CREATION_ALLOWED") == "true"
|
| 167 |
default_brain_fallback = os.getenv("DEFAULT_BRAIN_FALLBACK") == "true"
|
| 168 |
|
| 169 |
# Bypassing system routes --------------------------------
|
| 170 |
if request.url.path.startswith("/system") or request.url.path == "/":
|
| 171 |
return await call_next(request)
|
| 172 |
|
| 173 |
-
#
|
| 174 |
-
if brain_id
|
|
|
|
|
|
|
| 175 |
return error_response(
|
| 176 |
-
request,
|
| 177 |
-
|
| 178 |
-
detail="System brain is not allowed to be used.",
|
| 179 |
-
code="BRAIN_ID_RESERVED",
|
| 180 |
-
message="The system brain is reserved.",
|
| 181 |
-
resolution="Select a non-system application brain.",
|
| 182 |
)
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
return error_response(
|
| 187 |
-
request,
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
code="BRAIN_ID_INVALID",
|
| 191 |
-
message="The brain identifier is invalid.",
|
| 192 |
-
resolution="Use letters, digits, hyphens, or underscores.",
|
| 193 |
-
extra={"value": brain_id},
|
| 194 |
)
|
| 195 |
-
|
| 196 |
try:
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
)
|
| 200 |
-
if brain_id and not cached_brain_id:
|
| 201 |
-
stored_brain = data_adapter.get_brain(name_key=brain_id)
|
| 202 |
-
|
| 203 |
-
request.state.brain_id = brain_id
|
| 204 |
-
|
| 205 |
-
if not stored_brain and brain_creation_allowed:
|
| 206 |
-
new_brain = data_adapter.create_brain(name_key=brain_id)
|
| 207 |
-
cache_adapter.set(
|
| 208 |
-
key=f"brain:{brain_id}",
|
| 209 |
-
value=new_brain.id,
|
| 210 |
-
brain_id="system",
|
| 211 |
-
)
|
| 212 |
-
elif stored_brain:
|
| 213 |
-
cache_adapter.set(
|
| 214 |
-
key=f"brain:{brain_id}",
|
| 215 |
-
value=stored_brain.id,
|
| 216 |
-
brain_id="system",
|
| 217 |
-
)
|
| 218 |
-
elif not brain_id and default_brain_fallback:
|
| 219 |
-
default_brain = data_adapter.get_brain(name_key="default")
|
| 220 |
-
if not default_brain:
|
| 221 |
-
default_brain = data_adapter.create_brain(name_key="default")
|
| 222 |
-
cache_adapter.set(
|
| 223 |
-
key="brain:default",
|
| 224 |
-
value=default_brain.id,
|
| 225 |
-
brain_id="system",
|
| 226 |
-
)
|
| 227 |
-
request.state.brain_id = "default"
|
| 228 |
-
else:
|
| 229 |
-
cache_adapter.set(
|
| 230 |
-
key="brain:default",
|
| 231 |
-
value=default_brain.id,
|
| 232 |
-
brain_id="system",
|
| 233 |
-
)
|
| 234 |
-
request.state.brain_id = "default"
|
| 235 |
except Exception:
|
| 236 |
-
request.state.brain_id = brain_id or "default"
|
| 237 |
-
|
| 238 |
-
if getattr(request.state, "brain_id", None) is None:
|
| 239 |
return error_response(
|
| 240 |
-
request,
|
| 241 |
-
status_code=status.HTTP_406_NOT_ACCEPTABLE,
|
| 242 |
-
detail="Brain not found or creation is not allowed.",
|
| 243 |
)
|
| 244 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
return await call_next(request)
|
|
|
|
| 16 |
|
| 17 |
from fastapi import Request
|
| 18 |
from starlette.middleware.base import BaseHTTPMiddleware
|
|
|
|
| 19 |
|
| 20 |
from src.services.api.console_static import is_console_path
|
| 21 |
from src.services.api.errors import error_response
|
| 22 |
from src.services.data.main import data_adapter
|
| 23 |
+
from src.services.brain_identity import InvalidBrainIdError, validate_brain_id
|
| 24 |
from src.services.workspaces import WorkspaceScopeConflictError, resolve_workspace_scope
|
| 25 |
|
| 26 |
|
|
|
|
| 101 |
body_data = json.loads(body)
|
| 102 |
if isinstance(body_data, dict):
|
| 103 |
body_brain_id = body_data.get("brain_id")
|
| 104 |
+
if isinstance(body_brain_id, str):
|
| 105 |
body_brain_id = body_brain_id.rstrip()
|
| 106 |
except (json.JSONDecodeError, ValueError):
|
| 107 |
pass
|
|
|
|
| 162 |
brain_id, scope_error = await _get_brain_id()
|
| 163 |
if scope_error is not None:
|
| 164 |
return scope_error
|
|
|
|
| 165 |
default_brain_fallback = os.getenv("DEFAULT_BRAIN_FALLBACK") == "true"
|
| 166 |
|
| 167 |
# Bypassing system routes --------------------------------
|
| 168 |
if request.url.path.startswith("/system") or request.url.path == "/":
|
| 169 |
return await call_next(request)
|
| 170 |
|
| 171 |
+
# Scope resolution never provisions a brain, even with legacy creation flags.
|
| 172 |
+
if brain_id is None and default_brain_fallback:
|
| 173 |
+
brain_id = "default"
|
| 174 |
+
if brain_id is None:
|
| 175 |
return error_response(
|
| 176 |
+
request, status_code=400, detail="Brain ID is required.",
|
| 177 |
+
code="BRAIN_ID_REQUIRED",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
)
|
| 179 |
+
try:
|
| 180 |
+
brain_id = validate_brain_id(brain_id)
|
| 181 |
+
except InvalidBrainIdError as error:
|
| 182 |
return error_response(
|
| 183 |
+
request, status_code=400, detail=str(error),
|
| 184 |
+
code=error.code,
|
| 185 |
+
resolution="Use 1-128 ASCII letters, digits, hyphens, or underscores, starting with a letter or digit; system is reserved.",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
)
|
|
|
|
| 187 |
try:
|
| 188 |
+
# A stale cache must never override registry deletion or authorize storage.
|
| 189 |
+
stored_brain = data_adapter.get_brain(name_key=brain_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
except Exception:
|
|
|
|
|
|
|
|
|
|
| 191 |
return error_response(
|
| 192 |
+
request, status_code=503, detail="Brain registry unavailable",
|
|
|
|
|
|
|
| 193 |
)
|
| 194 |
+
if stored_brain is None:
|
| 195 |
+
request.state.brain_id = None
|
| 196 |
+
return error_response(
|
| 197 |
+
request, status_code=404, detail="Brain not found",
|
| 198 |
+
code="BRAIN_NOT_FOUND",
|
| 199 |
+
resolution="Create the brain explicitly or select a registered brain.",
|
| 200 |
+
)
|
| 201 |
+
request.state.brain_id = brain_id
|
| 202 |
return await call_next(request)
|
src/services/api/routes/public.py
CHANGED
|
@@ -4,6 +4,8 @@ from __future__ import annotations
|
|
| 4 |
|
| 5 |
import os
|
| 6 |
|
|
|
|
|
|
|
| 7 |
from fastapi import APIRouter, HTTPException, Query
|
| 8 |
from pydantic import BaseModel, Field
|
| 9 |
|
|
@@ -71,7 +73,9 @@ async def demo_search(
|
|
| 71 |
raise HTTPException(status_code=404, detail="Public demo search is disabled")
|
| 72 |
|
| 73 |
brain_id = os.getenv("PUBLIC_DEMO_BRAIN_ID", "agentdemo").strip()
|
| 74 |
-
|
|
|
|
|
|
|
| 75 |
raise HTTPException(
|
| 76 |
status_code=503,
|
| 77 |
detail="Public demo brain is not configured correctly",
|
|
|
|
| 4 |
|
| 5 |
import os
|
| 6 |
|
| 7 |
+
from src.services.brain_identity import InvalidBrainIdError, validate_brain_id
|
| 8 |
+
|
| 9 |
from fastapi import APIRouter, HTTPException, Query
|
| 10 |
from pydantic import BaseModel, Field
|
| 11 |
|
|
|
|
| 73 |
raise HTTPException(status_code=404, detail="Public demo search is disabled")
|
| 74 |
|
| 75 |
brain_id = os.getenv("PUBLIC_DEMO_BRAIN_ID", "agentdemo").strip()
|
| 76 |
+
try:
|
| 77 |
+
brain_id = validate_brain_id(brain_id)
|
| 78 |
+
except InvalidBrainIdError:
|
| 79 |
raise HTTPException(
|
| 80 |
status_code=503,
|
| 81 |
detail="Public demo brain is not configured correctly",
|
src/services/api/routes/system.py
CHANGED
|
@@ -62,9 +62,9 @@ async def clear_brain(brain_id: str, request: Request):
|
|
| 62 |
request,
|
| 63 |
status_code=400,
|
| 64 |
detail=str(error),
|
| 65 |
-
code=
|
| 66 |
message="The brain identifier is invalid.",
|
| 67 |
-
resolution="Use
|
| 68 |
)
|
| 69 |
except BrainNotFoundError as error:
|
| 70 |
return error_response(
|
|
@@ -112,9 +112,9 @@ async def delete_brain(brain_id: str, request: Request):
|
|
| 112 |
request,
|
| 113 |
status_code=400,
|
| 114 |
detail=str(error),
|
| 115 |
-
code=
|
| 116 |
message="The brain identifier is invalid.",
|
| 117 |
-
resolution="Use
|
| 118 |
)
|
| 119 |
except BrainLifecycleBusyError as error:
|
| 120 |
return error_response(
|
|
|
|
| 62 |
request,
|
| 63 |
status_code=400,
|
| 64 |
detail=str(error),
|
| 65 |
+
code=error.code,
|
| 66 |
message="The brain identifier is invalid.",
|
| 67 |
+
resolution="Use 1-128 ASCII letters, digits, hyphens, or underscores, starting with a letter or digit; system is reserved.",
|
| 68 |
)
|
| 69 |
except BrainNotFoundError as error:
|
| 70 |
return error_response(
|
|
|
|
| 112 |
request,
|
| 113 |
status_code=400,
|
| 114 |
detail=str(error),
|
| 115 |
+
code=error.code,
|
| 116 |
message="The brain identifier is invalid.",
|
| 117 |
+
resolution="Use 1-128 ASCII letters, digits, hyphens, or underscores, starting with a letter or digit; system is reserved.",
|
| 118 |
)
|
| 119 |
except BrainLifecycleBusyError as error:
|
| 120 |
return error_response(
|
src/services/brain_identity.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Canonical public brain identifiers and registry domain errors."""
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class InvalidBrainIdError(ValueError):
|
| 7 |
+
"""Invalid or reserved public brain identifier."""
|
| 8 |
+
code = "BRAIN_ID_INVALID"
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class ReservedBrainIdError(InvalidBrainIdError):
|
| 12 |
+
"""The identifier belongs to BrainAPI infrastructure."""
|
| 13 |
+
code = "BRAIN_ID_RESERVED"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class BrainNotFoundError(LookupError):
|
| 17 |
+
"""The canonical registry does not contain the requested brain."""
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class BrainAlreadyExistsError(ValueError):
|
| 21 |
+
"""A registry uniqueness constraint rejected brain creation."""
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def validate_brain_id(brain_id: str) -> str:
|
| 25 |
+
value = brain_id.strip() if isinstance(brain_id, str) else ""
|
| 26 |
+
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", value):
|
| 27 |
+
raise InvalidBrainIdError(
|
| 28 |
+
"brain_id must be 1-128 ASCII letters, digits, hyphens, or underscores "
|
| 29 |
+
"and start with a letter or digit"
|
| 30 |
+
)
|
| 31 |
+
if value.lower() == "system":
|
| 32 |
+
raise ReservedBrainIdError('brain_id "system" is reserved')
|
| 33 |
+
return value
|
src/services/brain_lifecycle.py
CHANGED
|
@@ -7,6 +7,10 @@ from dataclasses import dataclass
|
|
| 7 |
import logging
|
| 8 |
from typing import Any, Iterator, Protocol
|
| 9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
logger = logging.getLogger(__name__)
|
| 12 |
|
|
@@ -15,14 +19,6 @@ class BrainLifecycleError(RuntimeError):
|
|
| 15 |
"""Base class for lifecycle failures safe to map at the API boundary."""
|
| 16 |
|
| 17 |
|
| 18 |
-
class InvalidBrainIdError(BrainLifecycleError):
|
| 19 |
-
"""Raised when a lifecycle target is invalid or reserved."""
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
class BrainNotFoundError(BrainLifecycleError):
|
| 23 |
-
"""Raised when a clear operation targets an unknown brain."""
|
| 24 |
-
|
| 25 |
-
|
| 26 |
class BrainLifecycleBusyError(BrainLifecycleError):
|
| 27 |
"""Raised when another lifecycle operation already owns the brain lock."""
|
| 28 |
|
|
@@ -74,21 +70,6 @@ class BrainLifecycleResult:
|
|
| 74 |
}
|
| 75 |
|
| 76 |
|
| 77 |
-
def validate_brain_id(brain_id: str) -> str:
|
| 78 |
-
normalized = (brain_id or "").strip()
|
| 79 |
-
if (
|
| 80 |
-
not normalized
|
| 81 |
-
or not normalized.isalnum()
|
| 82 |
-
or not normalized[0].isalpha()
|
| 83 |
-
):
|
| 84 |
-
raise InvalidBrainIdError(
|
| 85 |
-
"brain_id must be alphanumeric and start with a letter"
|
| 86 |
-
)
|
| 87 |
-
if normalized.lower() == "system":
|
| 88 |
-
raise InvalidBrainIdError('brain_id "system" is reserved')
|
| 89 |
-
return normalized
|
| 90 |
-
|
| 91 |
-
|
| 92 |
class BrainLifecycleService:
|
| 93 |
"""Clear or delete one brain across the active storage profile."""
|
| 94 |
|
|
@@ -183,6 +164,10 @@ class BrainLifecycleService:
|
|
| 183 |
)
|
| 184 |
raise BrainCleanupError(brain_id, ["registry_cache"]) from None
|
| 185 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
self._data.delete_brain_registry(brain_id)
|
| 187 |
except Exception:
|
| 188 |
logger.exception("Failed to delete registry for brain %s", brain_id)
|
|
|
|
| 7 |
import logging
|
| 8 |
from typing import Any, Iterator, Protocol
|
| 9 |
|
| 10 |
+
from src.services.brain_identity import (
|
| 11 |
+
BrainNotFoundError, InvalidBrainIdError, validate_brain_id,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
|
|
|
|
| 19 |
"""Base class for lifecycle failures safe to map at the API boundary."""
|
| 20 |
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
class BrainLifecycleBusyError(BrainLifecycleError):
|
| 23 |
"""Raised when another lifecycle operation already owns the brain lock."""
|
| 24 |
|
|
|
|
| 70 |
}
|
| 71 |
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
class BrainLifecycleService:
|
| 74 |
"""Clear or delete one brain across the active storage profile."""
|
| 75 |
|
|
|
|
| 164 |
)
|
| 165 |
raise BrainCleanupError(brain_id, ["registry_cache"]) from None
|
| 166 |
try:
|
| 167 |
+
# PostgreSQL namespaces must disappear even when registry-less.
|
| 168 |
+
delete_storage = getattr(self._data, "delete_brain_storage", None)
|
| 169 |
+
if delete_storage is not None:
|
| 170 |
+
delete_storage(brain_id)
|
| 171 |
self._data.delete_brain_registry(brain_id)
|
| 172 |
except Exception:
|
| 173 |
logger.exception("Failed to delete registry for brain %s", brain_id)
|
src/services/mcp/utils.py
CHANGED
|
@@ -12,6 +12,7 @@ Modified By: Christian Nonis <alch.infoemail@gmail.com>
|
|
| 12 |
import secrets
|
| 13 |
|
| 14 |
from src.config import config
|
|
|
|
| 15 |
from src.services.data.main import data_adapter
|
| 16 |
|
| 17 |
|
|
@@ -25,6 +26,10 @@ def guard_brainpat(pat: str = None, target_brain: str = None) -> bool | str:
|
|
| 25 |
if pat and config.brainpat_token and secrets.compare_digest(
|
| 26 |
pat, config.brainpat_token
|
| 27 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
return True
|
| 29 |
|
| 30 |
brains = data_adapter.get_brains_list()
|
|
|
|
| 12 |
import secrets
|
| 13 |
|
| 14 |
from src.config import config
|
| 15 |
+
from src.services.brain_identity import BrainNotFoundError, validate_brain_id
|
| 16 |
from src.services.data.main import data_adapter
|
| 17 |
|
| 18 |
|
|
|
|
| 26 |
if pat and config.brainpat_token and secrets.compare_digest(
|
| 27 |
pat, config.brainpat_token
|
| 28 |
):
|
| 29 |
+
if target_brain:
|
| 30 |
+
target_brain = validate_brain_id(target_brain)
|
| 31 |
+
if data_adapter.get_brain(target_brain) is None:
|
| 32 |
+
raise BrainNotFoundError("Brain not found")
|
| 33 |
return True
|
| 34 |
|
| 35 |
brains = data_adapter.get_brains_list()
|
src/services/workspaces.py
CHANGED
|
@@ -9,6 +9,7 @@ from hashlib import sha1
|
|
| 9 |
from urllib.parse import quote, urlsplit
|
| 10 |
|
| 11 |
from src.constants.data import Brain, Workspace
|
|
|
|
| 12 |
|
| 13 |
SLUG_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
| 14 |
|
|
@@ -133,7 +134,11 @@ class WorkspaceService:
|
|
| 133 |
raise WorkspaceConflictError(
|
| 134 |
f'Brain "{slug}" already exists; run workspace bootstrap'
|
| 135 |
)
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
return self.repository.create_workspace(
|
| 138 |
Workspace(
|
| 139 |
brain_id=slug,
|
|
|
|
| 9 |
from urllib.parse import quote, urlsplit
|
| 10 |
|
| 11 |
from src.constants.data import Brain, Workspace
|
| 12 |
+
from src.services.brain_identity import BrainAlreadyExistsError, validate_brain_id
|
| 13 |
|
| 14 |
SLUG_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
| 15 |
|
|
|
|
| 134 |
raise WorkspaceConflictError(
|
| 135 |
f'Brain "{slug}" already exists; run workspace bootstrap'
|
| 136 |
)
|
| 137 |
+
validate_brain_id(slug)
|
| 138 |
+
try:
|
| 139 |
+
self.repository.create_brain(slug)
|
| 140 |
+
except BrainAlreadyExistsError:
|
| 141 |
+
raise WorkspaceConflictError("Brain already exists") from None
|
| 142 |
return self.repository.create_workspace(
|
| 143 |
Workspace(
|
| 144 |
brain_id=slug,
|
tests/integration/test_brain_registry_contract.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Opt-in gate against a disposable local HF CPU container, never the live Space."""
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import subprocess
|
| 5 |
+
|
| 6 |
+
import pytest
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@pytest.mark.integration
|
| 10 |
+
def test_real_registry_storage_contract():
|
| 11 |
+
container = os.getenv('BRAINAPI_REGISTRY_TEST_CONTAINER')
|
| 12 |
+
if not container:
|
| 13 |
+
pytest.skip('Set BRAINAPI_REGISTRY_TEST_CONTAINER to a disposable HF CPU container')
|
| 14 |
+
environment = dict(os.environ)
|
| 15 |
+
for selector in ('DOCKER_HOST', 'DOCKER_CONTEXT', 'DOCKER_TLS', 'DOCKER_TLS_VERIFY', 'DOCKER_CERT_PATH'):
|
| 16 |
+
environment.pop(selector, None)
|
| 17 |
+
result = subprocess.run(
|
| 18 |
+
['docker', '--host=unix:///var/run/docker.sock', 'exec', container,
|
| 19 |
+
'bash', '/app/entrypoint.sh', '--qualify-registry'],
|
| 20 |
+
env=environment, capture_output=True, text=True, timeout=300, check=True,
|
| 21 |
+
)
|
| 22 |
+
report = json.loads(result.stdout.strip().splitlines()[-1])
|
| 23 |
+
assert report['result'] == 'passed'
|
| 24 |
+
for operation in ('entities', 'search', 'context'):
|
| 25 |
+
assert report['unknown_' + operation] == '404 BRAIN_NOT_FOUND; storage/cache unchanged'
|
| 26 |
+
assert report['storage_duplicate_race'] == 'one create, one domain conflict'
|
| 27 |
+
assert report['orphan_cleanup'] == 'physical database removed without registry creation'
|
tests/test_agent_api.py
CHANGED
|
@@ -17,7 +17,6 @@ def agent_client(monkeypatch):
|
|
| 17 |
monkeypatch.setenv("BRAINPAT_TOKEN", "system-secret")
|
| 18 |
monkeypatch.setenv("PUBLIC_BASE_URL", "https://brain.example")
|
| 19 |
monkeypatch.setattr(brains, "data_adapter", data)
|
| 20 |
-
monkeypatch.setattr(brains, "cache_adapter", cache)
|
| 21 |
monkeypatch.setattr(auth, "data_adapter", data)
|
| 22 |
monkeypatch.setattr(auth, "cache_adapter", cache)
|
| 23 |
monkeypatch.setattr(agent, "data_adapter", data)
|
|
|
|
| 17 |
monkeypatch.setenv("BRAINPAT_TOKEN", "system-secret")
|
| 18 |
monkeypatch.setenv("PUBLIC_BASE_URL", "https://brain.example")
|
| 19 |
monkeypatch.setattr(brains, "data_adapter", data)
|
|
|
|
| 20 |
monkeypatch.setattr(auth, "data_adapter", data)
|
| 21 |
monkeypatch.setattr(auth, "cache_adapter", cache)
|
| 22 |
monkeypatch.setattr(agent, "data_adapter", data)
|
tests/test_agent_readiness_api.py
CHANGED
|
@@ -150,8 +150,8 @@ class StructuredErrorTests(unittest.TestCase):
|
|
| 150 |
return {"ok": True}
|
| 151 |
|
| 152 |
with patch(
|
| 153 |
-
"src.services.api.middlewares.brains.
|
| 154 |
-
side_effect=ConnectionError("
|
| 155 |
):
|
| 156 |
response = TestClient(app).get(
|
| 157 |
"/protected",
|
|
@@ -159,7 +159,7 @@ class StructuredErrorTests(unittest.TestCase):
|
|
| 159 |
)
|
| 160 |
self.assertEqual(response.status_code, 503)
|
| 161 |
self.assertEqual(response.json()["error"]["code"], "SERVICE_UNAVAILABLE")
|
| 162 |
-
self.assertNotIn("
|
| 163 |
|
| 164 |
|
| 165 |
class PublicApiTests(unittest.TestCase):
|
|
|
|
| 150 |
return {"ok": True}
|
| 151 |
|
| 152 |
with patch(
|
| 153 |
+
"src.services.api.middlewares.brains.data_adapter.get_brain",
|
| 154 |
+
side_effect=ConnectionError("registry secret"),
|
| 155 |
):
|
| 156 |
response = TestClient(app).get(
|
| 157 |
"/protected",
|
|
|
|
| 159 |
)
|
| 160 |
self.assertEqual(response.status_code, 503)
|
| 161 |
self.assertEqual(response.json()["error"]["code"], "SERVICE_UNAVAILABLE")
|
| 162 |
+
self.assertNotIn("registry secret", response.text)
|
| 163 |
|
| 164 |
|
| 165 |
class PublicApiTests(unittest.TestCase):
|
tests/test_brain_lifecycle.py
CHANGED
|
@@ -133,7 +133,7 @@ def test_partial_cleanup_keeps_registry_for_a_safe_retry():
|
|
| 133 |
assert "registry:delete:demo" not in events
|
| 134 |
|
| 135 |
|
| 136 |
-
@pytest.mark.parametrize("brain_id", ["", "
|
| 137 |
def test_lifecycle_rejects_invalid_or_reserved_brain_ids(brain_id: str):
|
| 138 |
with pytest.raises(InvalidBrainIdError):
|
| 139 |
validate_brain_id(brain_id)
|
|
|
|
| 133 |
assert "registry:delete:demo" not in events
|
| 134 |
|
| 135 |
|
| 136 |
+
@pytest.mark.parametrize("brain_id", ["", "bad id", "-bad", "system", "SYSTEM", "a" * 129])
|
| 137 |
def test_lifecycle_rejects_invalid_or_reserved_brain_ids(brain_id: str):
|
| 138 |
with pytest.raises(InvalidBrainIdError):
|
| 139 |
validate_brain_id(brain_id)
|
tests/test_brain_registry_contract.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Registry authority at the HTTP, PostgreSQL, and MCP boundaries."""
|
| 2 |
+
from contextlib import contextmanager
|
| 3 |
+
from copy import deepcopy
|
| 4 |
+
from types import SimpleNamespace
|
| 5 |
+
from unittest.mock import MagicMock
|
| 6 |
+
|
| 7 |
+
import psycopg2
|
| 8 |
+
import pytest
|
| 9 |
+
from fastapi import FastAPI
|
| 10 |
+
from fastapi.testclient import TestClient
|
| 11 |
+
|
| 12 |
+
from src.constants.data import Brain
|
| 13 |
+
from src.lib.postgresql import _provisioning, data, graph_store, vectors
|
| 14 |
+
from src.services.api.errors import install_error_handlers
|
| 15 |
+
from src.services.api.middlewares import auth, brains
|
| 16 |
+
from src.services.api.routes import retrieve, system
|
| 17 |
+
from src.services.api.controllers import system as system_controller
|
| 18 |
+
from src.services.brain_identity import (
|
| 19 |
+
BrainAlreadyExistsError, BrainNotFoundError, InvalidBrainIdError, validate_brain_id,
|
| 20 |
+
)
|
| 21 |
+
from src.services.brain_lifecycle import BrainLifecycleService
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
VALID_IDS = ['alpha', 'alpha123', 'lifeos-projects', 'probe_readonly_check_123', 'A1-test_name', '123', 'a' * 128]
|
| 25 |
+
INVALID_IDS = ['', '-bad', '_bad', 'bad id', 'a/b', 'éclair', 'a' * 129, 'system', 'SYSTEM']
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class Registry:
|
| 29 |
+
def __init__(self):
|
| 30 |
+
self.rows = {}
|
| 31 |
+
self.databases = set()
|
| 32 |
+
self.workspaces = {}
|
| 33 |
+
self.cache = {}
|
| 34 |
+
|
| 35 |
+
def get_brain(self, name_key):
|
| 36 |
+
return self.rows.get(name_key)
|
| 37 |
+
|
| 38 |
+
def create_brain(self, name_key):
|
| 39 |
+
if name_key in self.rows:
|
| 40 |
+
raise BrainAlreadyExistsError('Brain already exists')
|
| 41 |
+
brain = Brain(name_key=name_key)
|
| 42 |
+
self.rows[name_key] = brain
|
| 43 |
+
self.databases.add(name_key)
|
| 44 |
+
return brain
|
| 45 |
+
|
| 46 |
+
def get_brains_list(self):
|
| 47 |
+
return sorted(self.rows.values(), key=lambda b: b.name_key)
|
| 48 |
+
|
| 49 |
+
def clear_brain_data(self, brain_id):
|
| 50 |
+
pass
|
| 51 |
+
|
| 52 |
+
def delete_brain_storage(self, brain_id):
|
| 53 |
+
self.databases.discard(brain_id)
|
| 54 |
+
|
| 55 |
+
def delete_brain_registry(self, brain_id):
|
| 56 |
+
return self.rows.pop(brain_id, None) is not None
|
| 57 |
+
|
| 58 |
+
def clear_brain_registry_cache(self, brain_id):
|
| 59 |
+
self.cache.pop(brain_id, None)
|
| 60 |
+
|
| 61 |
+
def snapshot(self):
|
| 62 |
+
return deepcopy((self.rows, self.databases, self.workspaces, self.cache))
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@pytest.fixture
|
| 66 |
+
def api(monkeypatch):
|
| 67 |
+
registry = Registry()
|
| 68 |
+
monkeypatch.setenv('BRAINPAT_TOKEN', 'contract-system-pat')
|
| 69 |
+
monkeypatch.setattr(brains, 'data_adapter', registry)
|
| 70 |
+
monkeypatch.setattr(auth, 'data_adapter', registry)
|
| 71 |
+
monkeypatch.setattr(system_controller, 'data_adapter', registry)
|
| 72 |
+
lifecycle = BrainLifecycleService(data_client=registry, graph_client=registry,
|
| 73 |
+
vector_client=registry, cache_client=registry)
|
| 74 |
+
monkeypatch.setattr(system, 'get_brain_lifecycle_service', lambda: lifecycle)
|
| 75 |
+
# Controllers are isolated here; real storage + actual retrieval run in integration.
|
| 76 |
+
async def entities(*args, **kwargs):
|
| 77 |
+
return {'message': 'ok', 'entities': [], 'total': 0}
|
| 78 |
+
async def search(*args, **kwargs):
|
| 79 |
+
return {'hits': []}
|
| 80 |
+
async def context(*args, **kwargs):
|
| 81 |
+
return {'text_context': '', 'triples': []}
|
| 82 |
+
monkeypatch.setattr(retrieve, 'retrieve_get_entities_controller', entities)
|
| 83 |
+
monkeypatch.setattr(retrieve, 'search_controller', search)
|
| 84 |
+
monkeypatch.setattr(retrieve, 'retrieve_get_context_controller', context)
|
| 85 |
+
app = FastAPI()
|
| 86 |
+
install_error_handlers(app)
|
| 87 |
+
app.add_middleware(auth.BrainPATMiddleware)
|
| 88 |
+
app.add_middleware(brains.BrainMiddleware)
|
| 89 |
+
app.include_router(system.system_router)
|
| 90 |
+
app.include_router(retrieve.retrieve_router)
|
| 91 |
+
return TestClient(app), registry
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def read(client, kind, brain_id):
|
| 95 |
+
headers = {'BrainPAT': 'contract-system-pat'}
|
| 96 |
+
if kind == 'entities':
|
| 97 |
+
return client.get('/retrieve/entities', params={'brain_id': brain_id}, headers=headers)
|
| 98 |
+
return client.post('/retrieve/' + kind, json={
|
| 99 |
+
'brain_id': brain_id, 'query' if kind == 'search' else 'text': 'probe',
|
| 100 |
+
}, headers=headers)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@pytest.mark.parametrize('kind', ['entities', 'search', 'context'])
|
| 104 |
+
@pytest.mark.parametrize('creation_allowed', ['true', 'false'])
|
| 105 |
+
def test_unknown_retrieval_is_side_effect_free(api, monkeypatch, kind, creation_allowed):
|
| 106 |
+
client, registry = api
|
| 107 |
+
monkeypatch.setenv('BRAIN_CREATION_ALLOWED', creation_allowed)
|
| 108 |
+
# Deliberately stale positive cache cannot make an orphan authoritative.
|
| 109 |
+
registry.cache['probe_readonly_check_123'] = 'stale-id'
|
| 110 |
+
before = registry.snapshot()
|
| 111 |
+
response = read(client, kind, 'probe_readonly_check_123')
|
| 112 |
+
assert response.status_code == 404
|
| 113 |
+
assert response.json()['error']['code'] == 'BRAIN_NOT_FOUND'
|
| 114 |
+
assert registry.snapshot() == before
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
@pytest.mark.parametrize('kind', ['entities', 'search', 'context'])
|
| 118 |
+
def test_explicitly_provisioned_brain_reads_keep_identity(api, kind):
|
| 119 |
+
client, registry = api
|
| 120 |
+
created = client.post('/system/brains', json={'brain_id': 'known-brain'},
|
| 121 |
+
headers={'BrainPAT': 'contract-system-pat'})
|
| 122 |
+
assert created.status_code == 200
|
| 123 |
+
before = registry.snapshot()
|
| 124 |
+
response = read(client, kind, 'known-brain')
|
| 125 |
+
assert response.status_code == 200, response.text
|
| 126 |
+
assert registry.snapshot() == before
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
@pytest.mark.parametrize('brain_id', VALID_IDS)
|
| 130 |
+
def test_create_clear_delete_identifier_parity(api, brain_id):
|
| 131 |
+
client, registry = api
|
| 132 |
+
headers = {'BrainPAT': 'contract-system-pat'}
|
| 133 |
+
assert validate_brain_id(brain_id) == brain_id
|
| 134 |
+
assert client.post('/system/brains', headers=headers, json={'brain_id': brain_id}).status_code == 200
|
| 135 |
+
identity = (registry.rows[brain_id].id, registry.rows[brain_id].pat)
|
| 136 |
+
assert client.post(f'/system/brains/{brain_id}/clear', headers=headers).status_code == 200
|
| 137 |
+
assert (registry.rows[brain_id].id, registry.rows[brain_id].pat) == identity
|
| 138 |
+
assert client.delete(f'/system/brains/{brain_id}', headers=headers).status_code == 200
|
| 139 |
+
assert brain_id not in registry.rows and brain_id not in registry.databases
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
@pytest.mark.parametrize('brain_id', INVALID_IDS)
|
| 143 |
+
def test_invalid_identifiers_rejected_consistently(api, brain_id):
|
| 144 |
+
client, registry = api
|
| 145 |
+
with pytest.raises(InvalidBrainIdError) as invalid:
|
| 146 |
+
validate_brain_id(brain_id)
|
| 147 |
+
before = registry.snapshot()
|
| 148 |
+
response = client.post('/system/brains', headers={'BrainPAT': 'contract-system-pat'}, json={'brain_id': brain_id})
|
| 149 |
+
assert response.status_code == 400
|
| 150 |
+
assert response.json()['error']['code'] == invalid.value.code
|
| 151 |
+
assert registry.snapshot() == before
|
| 152 |
+
lifecycle = BrainLifecycleService(data_client=registry, graph_client=registry,
|
| 153 |
+
vector_client=registry, cache_client=registry)
|
| 154 |
+
for operation in (lifecycle.clear, lifecycle.delete):
|
| 155 |
+
with pytest.raises(InvalidBrainIdError):
|
| 156 |
+
operation(brain_id)
|
| 157 |
+
assert registry.snapshot() == before
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def test_duplicate_create_is_safe_conflict(api):
|
| 161 |
+
client, registry = api
|
| 162 |
+
headers = {'BrainPAT': 'contract-system-pat'}
|
| 163 |
+
assert client.post('/system/brains', headers=headers, json={'brain_id': 'alpha'}).status_code == 200
|
| 164 |
+
before = registry.snapshot()
|
| 165 |
+
response = client.post('/system/brains', headers=headers, json={'brain_id': 'alpha'})
|
| 166 |
+
assert response.status_code == 409
|
| 167 |
+
assert response.json()['error']['code'] == 'BRAIN_ALREADY_EXISTS'
|
| 168 |
+
assert registry.rows['alpha'].pat not in response.text
|
| 169 |
+
assert registry.snapshot() == before
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def test_orphan_delete_does_not_register_brain(api):
|
| 173 |
+
client, registry = api
|
| 174 |
+
registry.databases.add('orphan_storage')
|
| 175 |
+
response = client.delete('/system/brains/orphan_storage', headers={'BrainPAT': 'contract-system-pat'})
|
| 176 |
+
assert response.status_code == 200
|
| 177 |
+
assert response.json()['existed'] is False
|
| 178 |
+
assert not registry.rows and not registry.databases
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def test_postgres_unique_violation_maps_to_domain_conflict(monkeypatch):
|
| 182 |
+
client = data.PostgreSQLDataClient()
|
| 183 |
+
connection = MagicMock()
|
| 184 |
+
connection.cursor.return_value.__enter__.return_value.execute.side_effect = psycopg2.errors.UniqueViolation('secret DB details')
|
| 185 |
+
@contextmanager
|
| 186 |
+
def connect():
|
| 187 |
+
yield connection
|
| 188 |
+
monkeypatch.setattr(client, '_system_connection', connect)
|
| 189 |
+
provision = MagicMock()
|
| 190 |
+
monkeypatch.setattr(client, '_ensure_brain_schema', provision)
|
| 191 |
+
with pytest.raises(BrainAlreadyExistsError, match='^Brain already exists$'):
|
| 192 |
+
client.create_brain('alpha')
|
| 193 |
+
provision.assert_not_called()
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def test_registry_list_never_discovers_or_synthesizes_storage_brains(monkeypatch):
|
| 197 |
+
client = data.PostgreSQLDataClient()
|
| 198 |
+
registered = Brain(name_key='persisted')
|
| 199 |
+
connection = MagicMock()
|
| 200 |
+
connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [
|
| 201 |
+
{'document': registered.model_dump(mode='json')},
|
| 202 |
+
]
|
| 203 |
+
@contextmanager
|
| 204 |
+
def connect():
|
| 205 |
+
yield connection
|
| 206 |
+
monkeypatch.setattr(client, '_system_connection', connect)
|
| 207 |
+
discovery = MagicMock(return_value=['brain_persisted', 'brain_orphan'])
|
| 208 |
+
monkeypatch.setattr(_provisioning, 'list_brain_database_names', discovery)
|
| 209 |
+
assert client.get_brains_list() == client.get_brains_list() == [registered]
|
| 210 |
+
discovery.assert_not_called()
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
@pytest.mark.parametrize('backend', ['data', 'graph', 'vectors'])
|
| 214 |
+
def test_unknown_backend_read_cannot_create_storage_or_cache(monkeypatch, backend):
|
| 215 |
+
connection = MagicMock()
|
| 216 |
+
connection.cursor.return_value.__enter__.return_value.fetchone.return_value = None
|
| 217 |
+
pool = MagicMock()
|
| 218 |
+
pool.getconn.return_value = connection
|
| 219 |
+
monkeypatch.setattr(_provisioning.config, 'data_db', 'postgresql')
|
| 220 |
+
monkeypatch.setattr(_provisioning, 'get_system_pool', lambda: pool)
|
| 221 |
+
allocate = MagicMock(side_effect=AssertionError('brain connection allocated'))
|
| 222 |
+
provision = MagicMock(side_effect=AssertionError('database created'))
|
| 223 |
+
monkeypatch.setattr(_provisioning._registry, 'get', allocate)
|
| 224 |
+
monkeypatch.setattr(_provisioning, 'ensure_database_exists', provision)
|
| 225 |
+
if backend == 'data':
|
| 226 |
+
client = data.PostgreSQLDataClient()
|
| 227 |
+
operation = lambda: client.get_text_chunks(brain_id='unknown')
|
| 228 |
+
snapshots = lambda: deepcopy(client._initialized_brains)
|
| 229 |
+
elif backend == 'graph':
|
| 230 |
+
monkeypatch.setattr(graph_store.config, 'postgresql', SimpleNamespace(validate_credentials=lambda: None))
|
| 231 |
+
client = graph_store.PostgreSQLGraphStore()
|
| 232 |
+
operation = lambda: client.get_brain('unknown')
|
| 233 |
+
snapshots = lambda: deepcopy((client._schema_ready, client._brains))
|
| 234 |
+
else:
|
| 235 |
+
client = vectors.PostgreSQLVectorStoreClient()
|
| 236 |
+
operation = lambda: client.search_vectors([1, 0, 0], brain_id='unknown', store='data')
|
| 237 |
+
snapshots = lambda: deepcopy((client._brain_extensions_ready, client._initialized_stores))
|
| 238 |
+
before = snapshots()
|
| 239 |
+
with pytest.raises(BrainNotFoundError):
|
| 240 |
+
operation()
|
| 241 |
+
assert snapshots() == before
|
| 242 |
+
allocate.assert_not_called()
|
| 243 |
+
provision.assert_not_called()
|
| 244 |
+
statements = [c.args[0] for c in connection.cursor.return_value.__enter__.return_value.execute.call_args_list]
|
| 245 |
+
assert statements and all(s.startswith('SELECT ') for s in statements)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def test_mcp_system_pat_does_not_authorize_unknown_storage(monkeypatch):
|
| 249 |
+
from src.services.mcp import utils
|
| 250 |
+
registry = Registry()
|
| 251 |
+
monkeypatch.setattr(utils, 'data_adapter', registry)
|
| 252 |
+
monkeypatch.setattr(utils.config, 'brainpat_token', 'system-pat')
|
| 253 |
+
before = registry.snapshot()
|
| 254 |
+
with pytest.raises(BrainNotFoundError):
|
| 255 |
+
utils.guard_brainpat('system-pat', 'unknown')
|
| 256 |
+
assert registry.snapshot() == before
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def test_default_fallback_never_provisions_a_missing_default(api, monkeypatch):
|
| 260 |
+
client, registry = api
|
| 261 |
+
monkeypatch.setenv('DEFAULT_BRAIN_FALLBACK', 'true')
|
| 262 |
+
monkeypatch.setenv('BRAIN_CREATION_ALLOWED', 'true')
|
| 263 |
+
before = registry.snapshot()
|
| 264 |
+
response = client.post('/retrieve/search', json={'query': 'probe'}, headers={'BrainPAT': 'contract-system-pat'})
|
| 265 |
+
assert response.status_code == 404
|
| 266 |
+
assert response.json()['error']['code'] == 'BRAIN_NOT_FOUND'
|
| 267 |
+
assert registry.snapshot() == before
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def test_stale_graph_and_vector_cache_cannot_replace_registry(monkeypatch):
|
| 271 |
+
def missing(brain_id):
|
| 272 |
+
raise BrainNotFoundError('Brain not found')
|
| 273 |
+
monkeypatch.setattr(graph_store.config, 'postgresql', SimpleNamespace(validate_credentials=lambda: None))
|
| 274 |
+
monkeypatch.setattr(graph_store, 'require_registered_brain', missing)
|
| 275 |
+
graph = graph_store.PostgreSQLGraphStore()
|
| 276 |
+
graph._brains['deleted'] = graph_store._BrainGraph('deleted')
|
| 277 |
+
graph._schema_ready.add('deleted')
|
| 278 |
+
with pytest.raises(BrainNotFoundError):
|
| 279 |
+
graph.get_brain('deleted')
|
| 280 |
+
monkeypatch.setattr(vectors, 'require_registered_brain', missing)
|
| 281 |
+
vector = vectors.PostgreSQLVectorStoreClient()
|
| 282 |
+
vector._brain_extensions_ready.add('deleted')
|
| 283 |
+
with pytest.raises(BrainNotFoundError):
|
| 284 |
+
vector._ensure_extension('deleted')
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def test_mongo_unique_violation_maps_to_domain_conflict(monkeypatch):
|
| 288 |
+
from pymongo.errors import DuplicateKeyError
|
| 289 |
+
from src.config import config
|
| 290 |
+
monkeypatch.setattr(config, 'mongo', SimpleNamespace(
|
| 291 |
+
system_database='system', connection_string='mongodb://localhost:27017'))
|
| 292 |
+
from src.lib.mongo.client import MongoClient
|
| 293 |
+
client = MongoClient.__new__(MongoClient)
|
| 294 |
+
collection = MagicMock()
|
| 295 |
+
collection.insert_one.side_effect = DuplicateKeyError('secret database details')
|
| 296 |
+
client.get_collection = MagicMock(return_value=collection)
|
| 297 |
+
with pytest.raises(BrainAlreadyExistsError, match='^Brain already exists$'):
|
| 298 |
+
client.create_brain('alpha')
|
| 299 |
+
collection.create_index.assert_called_once_with('name_key', unique=True)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def test_mixed_profile_provisions_postgres_only_on_explicit_create(monkeypatch):
|
| 303 |
+
from src.adapters.data import DataAdapter
|
| 304 |
+
from src.config import config
|
| 305 |
+
from src.services.data import main
|
| 306 |
+
monkeypatch.setattr(config, 'data_db', 'mongo')
|
| 307 |
+
monkeypatch.setattr(config, 'vector_db', 'postgresql')
|
| 308 |
+
registry = Registry()
|
| 309 |
+
monkeypatch.setattr(main, 'data_adapter', registry)
|
| 310 |
+
adapter = DataAdapter()
|
| 311 |
+
adapter.add_client(registry)
|
| 312 |
+
allocate = MagicMock()
|
| 313 |
+
monkeypatch.setattr(_provisioning, 'ensure_brain_database', allocate)
|
| 314 |
+
with pytest.raises(BrainNotFoundError):
|
| 315 |
+
_provisioning.require_registered_brain('unknown')
|
| 316 |
+
allocate.assert_not_called()
|
| 317 |
+
brain = adapter.create_brain('mixed-profile')
|
| 318 |
+
allocate.assert_called_once_with(brain.name_key)
|
| 319 |
+
_provisioning.require_registered_brain('mixed-profile')
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def test_demo_seed_explicitly_provisions_a_missing_brain(monkeypatch):
|
| 323 |
+
from scripts import seed_public_demo
|
| 324 |
+
from unittest.mock import Mock
|
| 325 |
+
monkeypatch.setenv('BRAINPAT_TOKEN', 'seed-system-pat')
|
| 326 |
+
monkeypatch.setattr('sys.argv', ['seed_public_demo.py', '--brain', 'seed-demo'])
|
| 327 |
+
def response(status, payload=None, text=''):
|
| 328 |
+
result = Mock(status_code=status, text=text)
|
| 329 |
+
result.json.return_value = payload
|
| 330 |
+
return result
|
| 331 |
+
get = Mock(side_effect=[response(200, text='docs'),
|
| 332 |
+
response(404, {'error': {'code': 'BRAIN_NOT_FOUND'}}),
|
| 333 |
+
response(200, {'status': 'completed'})])
|
| 334 |
+
post = Mock(side_effect=[response(200, {'name_key': 'seed-demo'}),
|
| 335 |
+
response(200, {'task_id': 'seed-task'})])
|
| 336 |
+
monkeypatch.setattr(seed_public_demo.requests, 'get', get)
|
| 337 |
+
monkeypatch.setattr(seed_public_demo.requests, 'post', post)
|
| 338 |
+
assert seed_public_demo.main() == 0
|
| 339 |
+
assert post.call_args_list[0].args[0].endswith('/system/brains')
|
| 340 |
+
assert post.call_args_list[0].kwargs['json'] == {'brain_id': 'seed-demo'}
|
| 341 |
+
assert post.call_args_list[1].args[0].endswith('/ingest/')
|
tests/test_embeddinggemma2.py
CHANGED
|
@@ -113,6 +113,7 @@ def test_postgres_rejects_incompatible_index_without_dropping_vectors(gemma, mon
|
|
| 113 |
def connect(brain):
|
| 114 |
yield connection
|
| 115 |
monkeypatch.setattr(client, '_connection', connect)
|
|
|
|
| 116 |
monkeypatch.setitem(vectors.EMBEDDING_STORES_SIZES, 'data', 512)
|
| 117 |
with pytest.raises(EmbeddingProfileError, match='new empty brain/index'):
|
| 118 |
client._ensure_store('data', 'qualification')
|
|
@@ -129,6 +130,7 @@ def test_fresh_postgres_schema_records_profile_and_512_dimensions(gemma, monkeyp
|
|
| 129 |
def connect(brain):
|
| 130 |
yield connection
|
| 131 |
monkeypatch.setattr(client, '_connection', connect)
|
|
|
|
| 132 |
monkeypatch.setattr(vectors, 'EMBEDDING_STORES_SIZES', dict.fromkeys(
|
| 133 |
('nodes', 'triplets', 'observations', 'data', 'relationships'), 512))
|
| 134 |
for store in vectors.EMBEDDING_STORES_SIZES:
|
|
|
|
| 113 |
def connect(brain):
|
| 114 |
yield connection
|
| 115 |
monkeypatch.setattr(client, '_connection', connect)
|
| 116 |
+
monkeypatch.setattr(vectors, 'require_registered_brain', lambda brain_id: None)
|
| 117 |
monkeypatch.setitem(vectors.EMBEDDING_STORES_SIZES, 'data', 512)
|
| 118 |
with pytest.raises(EmbeddingProfileError, match='new empty brain/index'):
|
| 119 |
client._ensure_store('data', 'qualification')
|
|
|
|
| 130 |
def connect(brain):
|
| 131 |
yield connection
|
| 132 |
monkeypatch.setattr(client, '_connection', connect)
|
| 133 |
+
monkeypatch.setattr(vectors, 'require_registered_brain', lambda brain_id: None)
|
| 134 |
monkeypatch.setattr(vectors, 'EMBEDDING_STORES_SIZES', dict.fromkeys(
|
| 135 |
('nodes', 'triplets', 'observations', 'data', 'relationships'), 512))
|
| 136 |
for store in vectors.EMBEDDING_STORES_SIZES:
|
tests/test_graph_upsert_contract.py
CHANGED
|
@@ -170,7 +170,8 @@ class NetworkXUpsertContractTests(unittest.TestCase):
|
|
| 170 |
fresh.graph.add_edge("a", "b", key="r1", rel_type="KNOWS")
|
| 171 |
store._reload_brain_locked = MagicMock(return_value=fresh)
|
| 172 |
|
| 173 |
-
|
|
|
|
| 174 |
self.assertIs(loaded, fresh)
|
| 175 |
store._reload_brain_locked.assert_called_once_with("brain-a")
|
| 176 |
|
|
|
|
| 170 |
fresh.graph.add_edge("a", "b", key="r1", rel_type="KNOWS")
|
| 171 |
store._reload_brain_locked = MagicMock(return_value=fresh)
|
| 172 |
|
| 173 |
+
with patch("src.lib.postgresql.graph_store.require_registered_brain"):
|
| 174 |
+
loaded = store._load_brain("brain-a")
|
| 175 |
self.assertIs(loaded, fresh)
|
| 176 |
store._reload_brain_locked.assert_called_once_with("brain-a")
|
| 177 |
|
tests/test_retrieve_auth_isolation.py
CHANGED
|
@@ -7,7 +7,7 @@ client = TestClient(app)
|
|
| 7 |
|
| 8 |
@pytest.fixture
|
| 9 |
def mock_data_adapter():
|
| 10 |
-
with patch("src.services.api.middlewares.auth.data_adapter") as mock_da:
|
| 11 |
yield mock_da
|
| 12 |
|
| 13 |
@pytest.fixture
|
|
|
|
| 7 |
|
| 8 |
@pytest.fixture
|
| 9 |
def mock_data_adapter():
|
| 10 |
+
with patch("src.services.api.middlewares.auth.data_adapter") as mock_da, patch("src.services.api.middlewares.brains.data_adapter", mock_da):
|
| 11 |
yield mock_da
|
| 12 |
|
| 13 |
@pytest.fixture
|
tests/test_retrieve_http_contract.py
CHANGED
|
@@ -5,6 +5,14 @@ from src.services.api.app import app
|
|
| 5 |
|
| 6 |
client = TestClient(app)
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
def test_retrieve_search_post_unauthorized():
|
| 9 |
response = client.post("/retrieve/search", json={"query": "test"})
|
| 10 |
assert response.status_code == 401
|
|
|
|
| 5 |
|
| 6 |
client = TestClient(app)
|
| 7 |
|
| 8 |
+
@pytest.fixture(autouse=True)
|
| 9 |
+
def registered_scope(monkeypatch):
|
| 10 |
+
from types import SimpleNamespace
|
| 11 |
+
from src.services.api.middlewares import brains
|
| 12 |
+
monkeypatch.setattr(brains.data_adapter, 'get_brain', lambda name_key: SimpleNamespace(name_key=name_key))
|
| 13 |
+
monkeypatch.setenv('DEFAULT_BRAIN_FALLBACK', 'true')
|
| 14 |
+
|
| 15 |
+
|
| 16 |
def test_retrieve_search_post_unauthorized():
|
| 17 |
response = client.post("/retrieve/search", json={"query": "test"})
|
| 18 |
assert response.status_code == 401
|
tests/test_search_bm25.py
CHANGED
|
@@ -56,6 +56,13 @@ def _postgres_available() -> bool:
|
|
| 56 |
|
| 57 |
|
| 58 |
class OkapiBm25Tests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
def test_term_overlap_ranks_above_non_overlap(self):
|
| 60 |
overlap = okapi_bm25_document(
|
| 61 |
{"license": 2.0},
|
|
@@ -165,6 +172,8 @@ class OkapiBm25Tests(unittest.TestCase):
|
|
| 165 |
brain_id = "searchbench_bm25_unit"
|
| 166 |
with patch.object(pgdata.config, "search_enabled", True):
|
| 167 |
client = pgdata.PostgreSQLDataClient()
|
|
|
|
|
|
|
| 168 |
overlap = TextChunk(
|
| 169 |
id="overlap",
|
| 170 |
text="alice counseling license renewal board",
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
class OkapiBm25Tests(unittest.TestCase):
|
| 59 |
+
def setUp(self):
|
| 60 |
+
# These unit tests isolate FTS schema policy for registered namespaces.
|
| 61 |
+
for module in ('src.lib.postgresql.data', 'src.lib.postgresql.graph_store'):
|
| 62 |
+
guard = patch(module + '.require_registered_brain')
|
| 63 |
+
guard.start()
|
| 64 |
+
self.addCleanup(guard.stop)
|
| 65 |
+
|
| 66 |
def test_term_overlap_ranks_above_non_overlap(self):
|
| 67 |
overlap = okapi_bm25_document(
|
| 68 |
{"license": 2.0},
|
|
|
|
| 172 |
brain_id = "searchbench_bm25_unit"
|
| 173 |
with patch.object(pgdata.config, "search_enabled", True):
|
| 174 |
client = pgdata.PostgreSQLDataClient()
|
| 175 |
+
if client.get_brain(brain_id) is None:
|
| 176 |
+
client.create_brain(brain_id)
|
| 177 |
overlap = TextChunk(
|
| 178 |
id="overlap",
|
| 179 |
text="alice counseling license renewal board",
|
tests/test_workspace_facade.py
CHANGED
|
@@ -87,7 +87,6 @@ def facade_client(monkeypatch):
|
|
| 87 |
cache = FacadeCache()
|
| 88 |
monkeypatch.setenv("BRAINPAT_TOKEN", "system-secret")
|
| 89 |
monkeypatch.setattr(brains, "data_adapter", data)
|
| 90 |
-
monkeypatch.setattr(brains, "cache_adapter", cache)
|
| 91 |
monkeypatch.setattr(auth, "data_adapter", data)
|
| 92 |
monkeypatch.setattr(auth, "cache_adapter", cache)
|
| 93 |
monkeypatch.setattr(workspace_api, "data_adapter", data)
|
|
|
|
| 87 |
cache = FacadeCache()
|
| 88 |
monkeypatch.setenv("BRAINPAT_TOKEN", "system-secret")
|
| 89 |
monkeypatch.setattr(brains, "data_adapter", data)
|
|
|
|
| 90 |
monkeypatch.setattr(auth, "data_adapter", data)
|
| 91 |
monkeypatch.setattr(auth, "cache_adapter", cache)
|
| 92 |
monkeypatch.setattr(workspace_api, "data_adapter", data)
|
tests/test_workspace_plugin_contract.py
CHANGED
|
@@ -65,7 +65,6 @@ def test_workspace_plugin_route_receives_authoritative_brain(monkeypatch):
|
|
| 65 |
data, cache = FacadeData(), FacadeCache()
|
| 66 |
monkeypatch.setenv("BRAINPAT_TOKEN", "system-secret")
|
| 67 |
monkeypatch.setattr(brains, "data_adapter", data)
|
| 68 |
-
monkeypatch.setattr(brains, "cache_adapter", cache)
|
| 69 |
monkeypatch.setattr(auth, "data_adapter", data)
|
| 70 |
monkeypatch.setattr(auth, "cache_adapter", cache)
|
| 71 |
app.add_middleware(BrainPATMiddleware)
|
|
|
|
| 65 |
data, cache = FacadeData(), FacadeCache()
|
| 66 |
monkeypatch.setenv("BRAINPAT_TOKEN", "system-secret")
|
| 67 |
monkeypatch.setattr(brains, "data_adapter", data)
|
|
|
|
| 68 |
monkeypatch.setattr(auth, "data_adapter", data)
|
| 69 |
monkeypatch.setattr(auth, "cache_adapter", cache)
|
| 70 |
app.add_middleware(BrainPATMiddleware)
|