Spaces:
Sleeping
Sleeping
Merge pull request #18 from AlleksDev/search
Browse files- .env.example +14 -4
- README.md +48 -9
- app/modules/clubs/infrastructure/main_api_club_source.py +1 -1
- app/modules/events/infrastructure/main_api_event_source.py +41 -9
- app/modules/groups/infrastructure/main_api_group_source.py +1 -1
- app/modules/search/api/dependencies.py +29 -2
- app/modules/search/api/internal_auth.py +21 -0
- app/modules/search/api/internal_router.py +74 -0
- app/modules/search/api/internal_schemas.py +169 -0
- app/modules/search/api/public_router.py +86 -0
- app/modules/search/api/router.py +6 -79
- app/modules/search/api/schemas.py +8 -3
- app/modules/search/application/ports/query_embedding_provider.py +6 -0
- app/modules/search/application/use_cases/search_all.py +13 -6
- app/modules/search/application/use_cases/search_candidates.py +138 -0
- app/modules/search/domain/filters.py +27 -1
- app/modules/search/domain/models.py +20 -0
- app/modules/search/domain/query.py +12 -0
- app/modules/search/domain/relevance.py +69 -0
- app/modules/search/infrastructure/mock_provider.py +10 -2
- app/modules/search/infrastructure/pgvector_provider.py +21 -3
- app/modules/search/infrastructure/relevance_policy.py +33 -0
- app/modules/users/infrastructure/main_api_user_source.py +1 -1
- app/shared/config/settings.py +82 -9
- app/shared/search/source.py +8 -2
- app/shared/security/rate_limit.py +62 -3
- app/shared/vector_store/aws_pgvector.py +39 -4
- docs/api_endpoints.md +57 -2
- docs/global_search.md +140 -52
- scripts/calibrate_global_search_thresholds.py +215 -0
- sql/aws_pgvector_contract.sql +76 -4
- sql/migrations/20260712_01_global_search_candidate_filters.sql +413 -0
- sql/new_pgvector_schema.sql +76 -4
- sql/verify_global_search_candidate_filters.sql +186 -0
- tests/conftest.py +5 -0
- tests/test_feed_settings.py +34 -0
- tests/test_internal_search_candidates.py +115 -0
- tests/test_pgvector_readiness.py +22 -0
- tests/test_pgvector_search_provider.py +64 -0
- tests/test_rate_limit.py +43 -0
- tests/test_search_architecture.py +34 -0
- tests/test_search_candidates_use_case.py +110 -0
- tests/test_search_relevance.py +80 -0
- tests/test_search_resource_documents.py +37 -1
- tests/test_search_snapshot_sources.py +52 -0
- tests/test_search_threshold_calibration.py +16 -0
- tests/test_sql_contract.py +45 -0
.env.example
CHANGED
|
@@ -8,10 +8,10 @@ MAIN_API_PLACES_NEARBY_PATH=/api/v1/places/nearby
|
|
| 8 |
MAIN_API_POSTS_SNAPSHOT_PATH=/api/v1/internal/posts/snapshot
|
| 9 |
MAIN_API_POSTS_CHANGES_PATH=/api/v1/internal/posts/changes
|
| 10 |
MAIN_API_FEED_INTERACTIONS_PATH=/api/v1/internal/feed/interactions/changes
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
MAIN_API_INTERNAL_TOKEN=
|
| 16 |
MAIN_API_AUTH_TOKEN=
|
| 17 |
SEARCH_INTERNAL_TOKEN=
|
|
@@ -26,6 +26,12 @@ MAIN_API_PLACES_PAGINATION_MODE=cursor
|
|
| 26 |
MAIN_API_POSTS_PAGINATION_MODE=cursor
|
| 27 |
MAIN_API_SEARCH_PAGINATION_MODE=cursor
|
| 28 |
GLOBAL_SEARCH_NEARBY_BOOST=0.12
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
GROQ_API_KEY=
|
| 31 |
GROQ_MODEL=llama-3.1-8b-instant
|
|
@@ -58,6 +64,10 @@ SEMANTIC_RELEVANCE_THRESHOLD=0.50
|
|
| 58 |
|
| 59 |
LOG_LEVEL=INFO
|
| 60 |
REQUEST_TIMEOUT_SECONDS=10
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
LLM_TIMEOUT_SECONDS=12
|
| 62 |
MAX_LLM_CONCURRENT_REQUESTS=4
|
| 63 |
MAX_REQUEST_BODY_BYTES=262144
|
|
|
|
| 8 |
MAIN_API_POSTS_SNAPSHOT_PATH=/api/v1/internal/posts/snapshot
|
| 9 |
MAIN_API_POSTS_CHANGES_PATH=/api/v1/internal/posts/changes
|
| 10 |
MAIN_API_FEED_INTERACTIONS_PATH=/api/v1/internal/feed/interactions/changes
|
| 11 |
+
MAIN_API_USERS_SNAPSHOT_PATH=/api/v1/internal/search/users/snapshot
|
| 12 |
+
MAIN_API_CLUBS_SNAPSHOT_PATH=/api/v1/internal/search/clubs/snapshot
|
| 13 |
+
MAIN_API_GROUPS_SNAPSHOT_PATH=/api/v1/internal/search/groups/snapshot
|
| 14 |
+
MAIN_API_EVENTS_SNAPSHOT_PATH=/api/v1/internal/search/events/snapshot
|
| 15 |
MAIN_API_INTERNAL_TOKEN=
|
| 16 |
MAIN_API_AUTH_TOKEN=
|
| 17 |
SEARCH_INTERNAL_TOKEN=
|
|
|
|
| 26 |
MAIN_API_POSTS_PAGINATION_MODE=cursor
|
| 27 |
MAIN_API_SEARCH_PAGINATION_MODE=cursor
|
| 28 |
GLOBAL_SEARCH_NEARBY_BOOST=0.12
|
| 29 |
+
PUBLIC_GLOBAL_SEARCH_ENABLED=true
|
| 30 |
+
GLOBAL_SEARCH_MIN_SEMANTIC_SCORE=0.30
|
| 31 |
+
GLOBAL_SEARCH_MIN_LEXICAL_SCORE=0.05
|
| 32 |
+
# Overrides opcionales. Cada recurso incluido debe definir ambos valores.
|
| 33 |
+
GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON={}
|
| 34 |
+
GLOBAL_SEARCH_THRESHOLD_POLICY_VERSION=global-search-relevance-v1
|
| 35 |
|
| 36 |
GROQ_API_KEY=
|
| 37 |
GROQ_MODEL=llama-3.1-8b-instant
|
|
|
|
| 64 |
|
| 65 |
LOG_LEVEL=INFO
|
| 66 |
REQUEST_TIMEOUT_SECONDS=10
|
| 67 |
+
# Limites locales por IP y ruta. El gateway puede imponer limites distribuidos menores.
|
| 68 |
+
RATE_LIMIT_REQUESTS_PER_WINDOW=600
|
| 69 |
+
INTERNAL_RATE_LIMIT_REQUESTS_PER_WINDOW=120
|
| 70 |
+
RATE_LIMIT_WINDOW_SECONDS=60
|
| 71 |
LLM_TIMEOUT_SECONDS=12
|
| 72 |
MAX_LLM_CONCURRENT_REQUESTS=4
|
| 73 |
MAX_REQUEST_BODY_BYTES=262144
|
README.md
CHANGED
|
@@ -140,12 +140,36 @@ python -m app.jobs.sync_search_embeddings --resource all
|
|
| 140 |
Los jobs calculan un `content_hash` versionado con el contenido, modelo, version y
|
| 141 |
dimension. Un cambio de modelo fuerza la regeneracion aunque el texto no haya cambiado.
|
| 142 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
## Busqueda Global Hibrida
|
| 144 |
|
| 145 |
-
`POST /search`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
El query genera un solo embedding FastText. Cada proveedor combina similitud coseno con
|
| 147 |
-
full-text search de PostgreSQL mediante Reciprocal Rank Fusion
|
| 148 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
|
| 150 |
```json
|
| 151 |
{
|
|
@@ -157,12 +181,9 @@ un `top_results` diversificado mas una seccion independiente por recurso.
|
|
| 157 |
}
|
| 158 |
```
|
| 159 |
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
`requester_id`; las busquedas publicas no requieren autenticacion. Los grupos privados solo son recuperables
|
| 164 |
-
por su creador, miembros agregados o invitados. Clubes privados y eventos no publicos
|
| 165 |
-
permanecen ocultos mientras la API principal no entregue una lista de usuarios autorizados.
|
| 166 |
|
| 167 |
El indice de usuarios excluye correo, fecha de nacimiento, genero y ubicacion actual. Los
|
| 168 |
IDs de tags de eventos se conservan como metadatos; para aportar significado semantico la
|
|
@@ -197,6 +218,24 @@ sql/verify_post_feed_v2.sql
|
|
| 197 |
`sql/rollback_post_feed_v1.sql` es el rollback destructivo de emergencia. No se debe
|
| 198 |
ejecutar ninguna de estas migraciones desde la API ni desde un job.
|
| 199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
Para una BD vacía o una BD existente que ya use `VECTOR(300)`, puede ejecutarse
|
| 201 |
`sql/new_pgvector_schema.sql` desde pgAdmin. El archivo es convergente e incluye
|
| 202 |
el contrato base, feed V1, correcciones V2, propietarios y permisos. No convierte
|
|
|
|
| 140 |
Los jobs calculan un `content_hash` versionado con el contenido, modelo, version y
|
| 141 |
dimension. Un cambio de modelo fuerza la regeneracion aunque el texto no haya cambiado.
|
| 142 |
|
| 143 |
+
La sincronizacion de users, clubs, groups y events consume exclusivamente los snapshots
|
| 144 |
+
internos de la API principal:
|
| 145 |
+
|
| 146 |
+
```text
|
| 147 |
+
/api/v1/internal/search/users/snapshot
|
| 148 |
+
/api/v1/internal/search/clubs/snapshot
|
| 149 |
+
/api/v1/internal/search/groups/snapshot
|
| 150 |
+
/api/v1/internal/search/events/snapshot
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
Estos clientes requieren `MAIN_API_INTERNAL_TOKEN`; nunca usan un JWT de usuario ni
|
| 154 |
+
hacen fallback a `MAIN_API_AUTH_TOKEN`.
|
| 155 |
+
|
| 156 |
## Busqueda Global Hibrida
|
| 157 |
|
| 158 |
+
La app movil debe llamar `POST /api/v1/search` en la API principal con su token de
|
| 159 |
+
sesion. Go valida permisos y filtros de negocio, llama al endpoint interno
|
| 160 |
+
`POST /internal/search/candidates` de NLP, hidrata los IDs y devuelve los recursos
|
| 161 |
+
completos. NLP solo calcula candidatos y puntajes; no es la fuente de verdad ni devuelve
|
| 162 |
+
datos de presentacion.
|
| 163 |
+
|
| 164 |
El query genera un solo embedding FastText. Cada proveedor combina similitud coseno con
|
| 165 |
+
full-text search de PostgreSQL mediante Reciprocal Rank Fusion. Los resultados que no
|
| 166 |
+
superan el umbral semantico ni el lexical se descartan: no se agregan recursos de relleno.
|
| 167 |
+
Los eventos se descartan antes del ranking cuando
|
| 168 |
+
`start_time + duration_minutes <= as_of`.
|
| 169 |
+
|
| 170 |
+
`POST /search` permanece como endpoint publico de compatibilidad temporal. Puede
|
| 171 |
+
deshabilitarse con `PUBLIC_GLOBAL_SEARCH_ENABLED=false` cuando la app movil ya use solo
|
| 172 |
+
la API principal.
|
| 173 |
|
| 174 |
```json
|
| 175 |
{
|
|
|
|
| 181 |
}
|
| 182 |
```
|
| 183 |
|
| 184 |
+
La API principal obtiene la identidad desde el token de login; la app no debe enviar
|
| 185 |
+
`requester_id` como autoridad. El endpoint interno de NLP exige
|
| 186 |
+
`Authorization: Bearer <NLP_SERVICE_TOKEN>` y devuelve exclusivamente IDs y puntajes.
|
|
|
|
|
|
|
|
|
|
| 187 |
|
| 188 |
El indice de usuarios excluye correo, fecha de nacimiento, genero y ubicacion actual. Los
|
| 189 |
IDs de tags de eventos se conservan como metadatos; para aportar significado semantico la
|
|
|
|
| 218 |
`sql/rollback_post_feed_v1.sql` es el rollback destructivo de emergencia. No se debe
|
| 219 |
ejecutar ninguna de estas migraciones desde la API ni desde un job.
|
| 220 |
|
| 221 |
+
La busqueda coordinada por la API principal requiere, para una BD existente:
|
| 222 |
+
|
| 223 |
+
```text
|
| 224 |
+
sql/migrations/20260712_01_global_search_candidate_filters.sql
|
| 225 |
+
sql/verify_global_search_candidate_filters.sql
|
| 226 |
+
```
|
| 227 |
+
|
| 228 |
+
La migracion reemplaza de forma transaccional `search_resource_embeddings` y agrega dos
|
| 229 |
+
helpers de parseo seguro para timestamps y duraciones. No crea ni modifica tablas,
|
| 230 |
+
columnas o indices. El segundo archivo es de solo lectura, falla con una excepcion si
|
| 231 |
+
detecta deriva y debe ejecutarse despues para comprobar el contrato.
|
| 232 |
+
|
| 233 |
+
`/ready` exige la firma y los marcadores V2 de search; una funcion heredada ya no puede
|
| 234 |
+
declarar el servicio listo. Las conexiones y consultas pgvector usan
|
| 235 |
+
`REQUEST_TIMEOUT_SECONDS`, y las rutas tienen un limite local configurable con
|
| 236 |
+
`RATE_LIMIT_REQUESTS_PER_WINDOW`, `INTERNAL_RATE_LIMIT_REQUESTS_PER_WINDOW` y
|
| 237 |
+
`RATE_LIMIT_WINDOW_SECONDS`.
|
| 238 |
+
|
| 239 |
Para una BD vacía o una BD existente que ya use `VECTOR(300)`, puede ejecutarse
|
| 240 |
`sql/new_pgvector_schema.sql` desde pgAdmin. El archivo es convergente e incluye
|
| 241 |
el contrato base, feed V1, correcciones V2, propietarios y permisos. No convierte
|
app/modules/clubs/infrastructure/main_api_club_source.py
CHANGED
|
@@ -13,7 +13,7 @@ class MainApiClubsClient(PagedMainApiSearchClient):
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
-
path=settings.
|
| 17 |
collection_keys=("clubs",),
|
| 18 |
mapper=club_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
|
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
+
path=settings.main_api_clubs_snapshot_path,
|
| 17 |
collection_keys=("clubs",),
|
| 18 |
mapper=club_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
app/modules/events/infrastructure/main_api_event_source.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
from typing import Any, AsyncIterator
|
| 2 |
|
| 3 |
from app.modules.events.infrastructure.semantic_document import (
|
|
@@ -13,7 +14,7 @@ class MainApiEventsClient(PagedMainApiSearchClient):
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
-
path=settings.
|
| 17 |
collection_keys=("events",),
|
| 18 |
mapper=event_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
|
@@ -31,11 +32,16 @@ def event_to_source_record(event: dict[str, Any]) -> SearchSourceRecord | None:
|
|
| 31 |
return None
|
| 32 |
title = str(first_present(event, "title", "name", default="")).strip()
|
| 33 |
description = str(first_present(event, "description", "summary", default="")).strip()
|
| 34 |
-
|
| 35 |
-
tag_ids = as_text_list(
|
| 36 |
# Only textual tag names are semantic. UUID/numeric IDs stay as filters/metadata.
|
| 37 |
-
tag_names = [tag for tag in
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
is_public = bool(first_present(event, "is_public", "isPublic", default=True))
|
| 40 |
document = build_event_search_document(title, tag_names, description)
|
| 41 |
return make_search_record(
|
|
@@ -49,16 +55,15 @@ def event_to_source_record(event: dict[str, Any]) -> SearchSourceRecord | None:
|
|
| 49 |
"description": description[:300],
|
| 50 |
"club_id": first_present(event, "club_id", "clubId"),
|
| 51 |
"place_id": first_present(event, "place_id", "placeId"),
|
| 52 |
-
"start_time":
|
| 53 |
-
"duration_minutes":
|
| 54 |
-
event, "duration_minutes", "durationMinutes"
|
| 55 |
-
),
|
| 56 |
"max_attendees": first_present(event, "max_attendees", "maxAttendees"),
|
| 57 |
"is_public": is_public,
|
| 58 |
"cover_image_url": first_present(event, "cover_image_url", "coverImageUrl"),
|
| 59 |
"tags": tag_names,
|
| 60 |
"tag_ids": tag_ids,
|
| 61 |
"is_active": is_active,
|
|
|
|
| 62 |
"document_version": EVENT_SEARCH_DOCUMENT_VERSION,
|
| 63 |
},
|
| 64 |
)
|
|
@@ -67,3 +72,30 @@ def event_to_source_record(event: dict[str, Any]) -> SearchSourceRecord | None:
|
|
| 67 |
def _looks_like_identifier(value: str) -> bool:
|
| 68 |
compact = value.replace("-", "")
|
| 69 |
return compact.isdigit() or (len(compact) == 32 and all(c in "0123456789abcdefABCDEF" for c in compact))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datetime import UTC, datetime
|
| 2 |
from typing import Any, AsyncIterator
|
| 3 |
|
| 4 |
from app.modules.events.infrastructure.semantic_document import (
|
|
|
|
| 14 |
def __init__(self, settings: Settings) -> None:
|
| 15 |
super().__init__(
|
| 16 |
settings=settings,
|
| 17 |
+
path=settings.main_api_events_snapshot_path,
|
| 18 |
collection_keys=("events",),
|
| 19 |
mapper=event_to_source_record,
|
| 20 |
page_limit=settings.main_api_search_page_limit,
|
|
|
|
| 32 |
return None
|
| 33 |
title = str(first_present(event, "title", "name", default="")).strip()
|
| 34 |
description = str(first_present(event, "description", "summary", default="")).strip()
|
| 35 |
+
tag_names = as_text_list(first_present(event, "tags", default=[]))
|
| 36 |
+
tag_ids = as_text_list(first_present(event, "tag_ids", "tagIds", default=[]))
|
| 37 |
# Only textual tag names are semantic. UUID/numeric IDs stay as filters/metadata.
|
| 38 |
+
tag_names = [tag for tag in tag_names if not _looks_like_identifier(tag)]
|
| 39 |
+
start_time, duration_minutes = _normalized_temporal_metadata(event)
|
| 40 |
+
temporal_metadata_valid = start_time is not None and duration_minutes is not None
|
| 41 |
+
is_active = (
|
| 42 |
+
bool(first_present(event, "is_active", "isActive", default=True))
|
| 43 |
+
and temporal_metadata_valid
|
| 44 |
+
)
|
| 45 |
is_public = bool(first_present(event, "is_public", "isPublic", default=True))
|
| 46 |
document = build_event_search_document(title, tag_names, description)
|
| 47 |
return make_search_record(
|
|
|
|
| 55 |
"description": description[:300],
|
| 56 |
"club_id": first_present(event, "club_id", "clubId"),
|
| 57 |
"place_id": first_present(event, "place_id", "placeId"),
|
| 58 |
+
"start_time": start_time,
|
| 59 |
+
"duration_minutes": duration_minutes,
|
|
|
|
|
|
|
| 60 |
"max_attendees": first_present(event, "max_attendees", "maxAttendees"),
|
| 61 |
"is_public": is_public,
|
| 62 |
"cover_image_url": first_present(event, "cover_image_url", "coverImageUrl"),
|
| 63 |
"tags": tag_names,
|
| 64 |
"tag_ids": tag_ids,
|
| 65 |
"is_active": is_active,
|
| 66 |
+
"temporal_metadata_valid": temporal_metadata_valid,
|
| 67 |
"document_version": EVENT_SEARCH_DOCUMENT_VERSION,
|
| 68 |
},
|
| 69 |
)
|
|
|
|
| 72 |
def _looks_like_identifier(value: str) -> bool:
|
| 73 |
compact = value.replace("-", "")
|
| 74 |
return compact.isdigit() or (len(compact) == 32 and all(c in "0123456789abcdefABCDEF" for c in compact))
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _normalized_temporal_metadata(
|
| 78 |
+
event: dict[str, Any],
|
| 79 |
+
) -> tuple[str | None, int | None]:
|
| 80 |
+
raw_start = first_present(event, "start_time", "startTime")
|
| 81 |
+
if not isinstance(raw_start, str) or not raw_start.strip():
|
| 82 |
+
return None, None
|
| 83 |
+
try:
|
| 84 |
+
parsed_start = datetime.fromisoformat(
|
| 85 |
+
raw_start.strip().replace("Z", "+00:00")
|
| 86 |
+
)
|
| 87 |
+
except ValueError:
|
| 88 |
+
return None, None
|
| 89 |
+
if parsed_start.utcoffset() is None:
|
| 90 |
+
return None, None
|
| 91 |
+
|
| 92 |
+
raw_duration = first_present(event, "duration_minutes", "durationMinutes")
|
| 93 |
+
if isinstance(raw_duration, bool):
|
| 94 |
+
return None, None
|
| 95 |
+
try:
|
| 96 |
+
parsed_duration = int(raw_duration)
|
| 97 |
+
except (TypeError, ValueError):
|
| 98 |
+
return None, None
|
| 99 |
+
if parsed_duration <= 0 or parsed_duration > 2_147_483_647:
|
| 100 |
+
return None, None
|
| 101 |
+
return parsed_start.astimezone(UTC).isoformat(), parsed_duration
|
app/modules/groups/infrastructure/main_api_group_source.py
CHANGED
|
@@ -13,7 +13,7 @@ class MainApiGroupsClient(PagedMainApiSearchClient):
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
-
path=settings.
|
| 17 |
collection_keys=("groups",),
|
| 18 |
mapper=group_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
|
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
+
path=settings.main_api_groups_snapshot_path,
|
| 17 |
collection_keys=("groups",),
|
| 18 |
mapper=group_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
app/modules/search/api/dependencies.py
CHANGED
|
@@ -2,6 +2,9 @@ from functools import lru_cache
|
|
| 2 |
|
| 3 |
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 4 |
from app.modules.search.application.use_cases.search_all import SearchAllUseCase
|
|
|
|
|
|
|
|
|
|
| 5 |
from app.modules.search.domain.models import ALL_SEARCH_RESOURCE_TYPES, SearchResourceType
|
| 6 |
from app.modules.search.infrastructure.mock_provider import (
|
| 7 |
MockHybridSearchProvider,
|
|
@@ -11,6 +14,8 @@ from app.modules.search.infrastructure.pgvector_provider import (
|
|
| 11 |
PgvectorHybridSearchProvider,
|
| 12 |
PgvectorPlaceSearchProvider,
|
| 13 |
)
|
|
|
|
|
|
|
| 14 |
from app.modules.places.infrastructure.main_api_nearby_place_provider import (
|
| 15 |
MainApiNearbyPlaceProvider,
|
| 16 |
)
|
|
@@ -19,17 +24,27 @@ from app.shared.dependencies import get_embedding_provider
|
|
| 19 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
| 20 |
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
@lru_cache
|
| 23 |
def get_global_search_providers() -> tuple[SearchProvider, ...]:
|
| 24 |
settings = get_settings()
|
| 25 |
embedding_provider = get_embedding_provider()
|
|
|
|
| 26 |
if settings.vector_store_provider == "aws_pgvector":
|
| 27 |
vector_client = AwsPgvectorClient(settings, role="reader")
|
| 28 |
return tuple(
|
| 29 |
(
|
| 30 |
-
PgvectorPlaceSearchProvider(vector_client)
|
| 31 |
if resource_type == SearchResourceType.PLACES
|
| 32 |
-
else PgvectorHybridSearchProvider(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
)
|
| 34 |
for resource_type in ALL_SEARCH_RESOURCE_TYPES
|
| 35 |
)
|
|
@@ -40,6 +55,7 @@ def get_global_search_providers() -> tuple[SearchProvider, ...]:
|
|
| 40 |
resource_type=resource_type,
|
| 41 |
embedding_provider=embedding_provider,
|
| 42 |
documents=documents[resource_type],
|
|
|
|
| 43 |
)
|
| 44 |
for resource_type in ALL_SEARCH_RESOURCE_TYPES
|
| 45 |
)
|
|
@@ -52,3 +68,14 @@ def get_search_all_use_case() -> SearchAllUseCase:
|
|
| 52 |
providers=list(get_global_search_providers()),
|
| 53 |
nearby_place_provider=MainApiNearbyPlaceProvider(get_settings()),
|
| 54 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 4 |
from app.modules.search.application.use_cases.search_all import SearchAllUseCase
|
| 5 |
+
from app.modules.search.application.use_cases.search_candidates import (
|
| 6 |
+
SearchCandidatesUseCase,
|
| 7 |
+
)
|
| 8 |
from app.modules.search.domain.models import ALL_SEARCH_RESOURCE_TYPES, SearchResourceType
|
| 9 |
from app.modules.search.infrastructure.mock_provider import (
|
| 10 |
MockHybridSearchProvider,
|
|
|
|
| 14 |
PgvectorHybridSearchProvider,
|
| 15 |
PgvectorPlaceSearchProvider,
|
| 16 |
)
|
| 17 |
+
from app.modules.search.infrastructure.relevance_policy import build_relevance_policy
|
| 18 |
+
from app.modules.search.domain.relevance import SearchRelevancePolicy
|
| 19 |
from app.modules.places.infrastructure.main_api_nearby_place_provider import (
|
| 20 |
MainApiNearbyPlaceProvider,
|
| 21 |
)
|
|
|
|
| 24 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
| 25 |
|
| 26 |
|
| 27 |
+
@lru_cache
|
| 28 |
+
def get_search_relevance_policy() -> SearchRelevancePolicy:
|
| 29 |
+
return build_relevance_policy(get_settings())
|
| 30 |
+
|
| 31 |
+
|
| 32 |
@lru_cache
|
| 33 |
def get_global_search_providers() -> tuple[SearchProvider, ...]:
|
| 34 |
settings = get_settings()
|
| 35 |
embedding_provider = get_embedding_provider()
|
| 36 |
+
relevance_policy = get_search_relevance_policy()
|
| 37 |
if settings.vector_store_provider == "aws_pgvector":
|
| 38 |
vector_client = AwsPgvectorClient(settings, role="reader")
|
| 39 |
return tuple(
|
| 40 |
(
|
| 41 |
+
PgvectorPlaceSearchProvider(vector_client, relevance_policy)
|
| 42 |
if resource_type == SearchResourceType.PLACES
|
| 43 |
+
else PgvectorHybridSearchProvider(
|
| 44 |
+
resource_type,
|
| 45 |
+
vector_client,
|
| 46 |
+
relevance_policy,
|
| 47 |
+
)
|
| 48 |
)
|
| 49 |
for resource_type in ALL_SEARCH_RESOURCE_TYPES
|
| 50 |
)
|
|
|
|
| 55 |
resource_type=resource_type,
|
| 56 |
embedding_provider=embedding_provider,
|
| 57 |
documents=documents[resource_type],
|
| 58 |
+
relevance_policy=relevance_policy,
|
| 59 |
)
|
| 60 |
for resource_type in ALL_SEARCH_RESOURCE_TYPES
|
| 61 |
)
|
|
|
|
| 68 |
providers=list(get_global_search_providers()),
|
| 69 |
nearby_place_provider=MainApiNearbyPlaceProvider(get_settings()),
|
| 70 |
)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@lru_cache
|
| 74 |
+
def get_search_candidates_use_case() -> SearchCandidatesUseCase:
|
| 75 |
+
policy = get_search_relevance_policy()
|
| 76 |
+
return SearchCandidatesUseCase(
|
| 77 |
+
embedding_provider=get_embedding_provider(),
|
| 78 |
+
providers=list(get_global_search_providers()),
|
| 79 |
+
policy_version=policy.version,
|
| 80 |
+
nearby_place_provider=MainApiNearbyPlaceProvider(get_settings()),
|
| 81 |
+
)
|
app/modules/search/api/internal_auth.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hmac
|
| 2 |
+
|
| 3 |
+
from fastapi import Header, HTTPException, status
|
| 4 |
+
|
| 5 |
+
from app.shared.config.settings import get_settings
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def require_search_service_token(
|
| 9 |
+
authorization: str | None = Header(default=None, alias="Authorization"),
|
| 10 |
+
) -> None:
|
| 11 |
+
expected = get_settings().post_feed_internal_token
|
| 12 |
+
if not expected:
|
| 13 |
+
raise HTTPException(
|
| 14 |
+
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
| 15 |
+
detail="NLP_SERVICE_TOKEN no configurado",
|
| 16 |
+
)
|
| 17 |
+
scheme, separator, received = (authorization or "").partition(" ")
|
| 18 |
+
if not separator or scheme.casefold() != "bearer" or not received.strip():
|
| 19 |
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
|
| 20 |
+
if not hmac.compare_digest(received.strip(), expected):
|
| 21 |
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
|
app/modules/search/api/internal_router.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
|
| 3 |
+
from fastapi import APIRouter, Depends, HTTPException
|
| 4 |
+
|
| 5 |
+
from app.modules.search.api.cursor import (
|
| 6 |
+
build_search_cursor_context,
|
| 7 |
+
decode_search_cursor,
|
| 8 |
+
)
|
| 9 |
+
from app.modules.search.api.dependencies import get_search_candidates_use_case
|
| 10 |
+
from app.modules.search.api.internal_auth import require_search_service_token
|
| 11 |
+
from app.modules.search.api.internal_schemas import (
|
| 12 |
+
InternalSearchCandidatesRequest,
|
| 13 |
+
InternalSearchCandidatesResponse,
|
| 14 |
+
candidates_result_to_schema,
|
| 15 |
+
)
|
| 16 |
+
from app.modules.search.application.use_cases.search_candidates import (
|
| 17 |
+
SearchCandidatesUseCase,
|
| 18 |
+
)
|
| 19 |
+
from app.shared.config.settings import get_settings
|
| 20 |
+
from app.shared.security.rate_limit import rate_limit_placeholder
|
| 21 |
+
|
| 22 |
+
router = APIRouter(
|
| 23 |
+
prefix="/internal/search",
|
| 24 |
+
tags=["internal-search"],
|
| 25 |
+
dependencies=[
|
| 26 |
+
Depends(rate_limit_placeholder),
|
| 27 |
+
Depends(require_search_service_token),
|
| 28 |
+
],
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@router.post(
|
| 33 |
+
"/candidates",
|
| 34 |
+
response_model=InternalSearchCandidatesResponse,
|
| 35 |
+
)
|
| 36 |
+
async def search_candidates(
|
| 37 |
+
payload: InternalSearchCandidatesRequest,
|
| 38 |
+
use_case: SearchCandidatesUseCase = Depends(get_search_candidates_use_case),
|
| 39 |
+
) -> InternalSearchCandidatesResponse:
|
| 40 |
+
settings = get_settings()
|
| 41 |
+
cursor_context = build_search_cursor_context(
|
| 42 |
+
payload.cursor_context_payload(
|
| 43 |
+
nearby_boost=settings.global_search_nearby_boost,
|
| 44 |
+
policy_version=settings.global_search_threshold_policy_version,
|
| 45 |
+
)
|
| 46 |
+
)
|
| 47 |
+
try:
|
| 48 |
+
offsets = {
|
| 49 |
+
resource_type: decode_search_cursor(
|
| 50 |
+
cursor=cursor,
|
| 51 |
+
resource_type=resource_type,
|
| 52 |
+
context_fingerprint=cursor_context,
|
| 53 |
+
)
|
| 54 |
+
for resource_type, cursor in payload.cursors.items()
|
| 55 |
+
}
|
| 56 |
+
except ValueError as exc:
|
| 57 |
+
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
| 58 |
+
try:
|
| 59 |
+
async with asyncio.timeout(settings.request_timeout_seconds):
|
| 60 |
+
result = await use_case.execute(
|
| 61 |
+
query=payload.query,
|
| 62 |
+
resource_types=tuple(payload.resource_types),
|
| 63 |
+
candidate_limit_per_type=payload.candidate_limit_per_type,
|
| 64 |
+
as_of=payload.as_of,
|
| 65 |
+
offsets=offsets,
|
| 66 |
+
criteria=payload.to_domain_criteria(settings.global_search_nearby_boost),
|
| 67 |
+
cursor_context=cursor_context,
|
| 68 |
+
)
|
| 69 |
+
except TimeoutError as exc:
|
| 70 |
+
raise HTTPException(
|
| 71 |
+
status_code=503,
|
| 72 |
+
detail="Internal search timed out",
|
| 73 |
+
) from exc
|
| 74 |
+
return candidates_result_to_schema(result)
|
app/modules/search/api/internal_schemas.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datetime import UTC, datetime
|
| 2 |
+
|
| 3 |
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
| 4 |
+
|
| 5 |
+
from app.modules.search.api.cursor import MAX_SEARCH_OFFSET, encode_search_cursor
|
| 6 |
+
from app.modules.search.api.schemas import (
|
| 7 |
+
GlobalSearchFiltersSchema,
|
| 8 |
+
SearchLocationSchema,
|
| 9 |
+
)
|
| 10 |
+
from app.modules.search.domain.filters import SearchCriteria
|
| 11 |
+
from app.modules.search.domain.models import (
|
| 12 |
+
SearchCandidate,
|
| 13 |
+
SearchCandidatesResult,
|
| 14 |
+
SearchResourceType,
|
| 15 |
+
)
|
| 16 |
+
from app.modules.search.domain.query import normalize_search_query
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class InternalSearchCandidatesRequest(BaseModel):
|
| 20 |
+
model_config = ConfigDict(extra="forbid")
|
| 21 |
+
|
| 22 |
+
query: str = Field(..., min_length=1, max_length=500)
|
| 23 |
+
resource_types: list[SearchResourceType] = Field(..., min_length=1, max_length=6)
|
| 24 |
+
candidate_limit_per_type: int = Field(default=20, ge=1, le=500)
|
| 25 |
+
top_limit: int = Field(default=30, ge=1, le=3000)
|
| 26 |
+
cursors: dict[SearchResourceType, str] = Field(default_factory=dict)
|
| 27 |
+
as_of: datetime
|
| 28 |
+
location: SearchLocationSchema | None = None
|
| 29 |
+
filters: GlobalSearchFiltersSchema = Field(default_factory=GlobalSearchFiltersSchema)
|
| 30 |
+
|
| 31 |
+
@field_validator("query")
|
| 32 |
+
@classmethod
|
| 33 |
+
def validate_query(cls, value: str) -> str:
|
| 34 |
+
if not normalize_search_query(value):
|
| 35 |
+
raise ValueError("query must not be blank")
|
| 36 |
+
return value
|
| 37 |
+
|
| 38 |
+
@model_validator(mode="after")
|
| 39 |
+
def validate_context(self) -> "InternalSearchCandidatesRequest":
|
| 40 |
+
if self.as_of.utcoffset() is None:
|
| 41 |
+
raise ValueError("as_of must include a timezone")
|
| 42 |
+
if len(self.resource_types) != len(set(self.resource_types)):
|
| 43 |
+
raise ValueError("resource_types must not contain duplicates")
|
| 44 |
+
unexpected = set(self.cursors) - set(self.resource_types)
|
| 45 |
+
if unexpected:
|
| 46 |
+
names = ", ".join(sorted(item.value for item in unexpected))
|
| 47 |
+
raise ValueError(f"cursor resources were not requested: {names}")
|
| 48 |
+
return self
|
| 49 |
+
|
| 50 |
+
def to_domain_criteria(self, nearby_boost: float) -> SearchCriteria:
|
| 51 |
+
return SearchCriteria(
|
| 52 |
+
filters=self.filters.to_domain(),
|
| 53 |
+
location=self.location.to_domain() if self.location else None,
|
| 54 |
+
nearby_boost=nearby_boost,
|
| 55 |
+
event_active_at=self.as_of,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
def cursor_context_payload(
|
| 59 |
+
self,
|
| 60 |
+
nearby_boost: float,
|
| 61 |
+
policy_version: str,
|
| 62 |
+
) -> dict[str, object]:
|
| 63 |
+
filters = self.filters.model_dump(mode="json")
|
| 64 |
+
for key in ("categories", "price_ranges", "tags", "user_roles"):
|
| 65 |
+
filters[key] = sorted(
|
| 66 |
+
{
|
| 67 |
+
str(value).strip().casefold()
|
| 68 |
+
for value in filters[key]
|
| 69 |
+
if str(value).strip()
|
| 70 |
+
}
|
| 71 |
+
)
|
| 72 |
+
for key in ("city", "state"):
|
| 73 |
+
if filters[key]:
|
| 74 |
+
filters[key] = str(filters[key]).strip().casefold()
|
| 75 |
+
for key in ("published_from", "published_to", "event_from", "event_to"):
|
| 76 |
+
value = getattr(self.filters, key)
|
| 77 |
+
filters[key] = value.astimezone(UTC).isoformat() if value else None
|
| 78 |
+
return {
|
| 79 |
+
"query": normalize_search_query(self.query),
|
| 80 |
+
"candidate_limit_per_type": self.candidate_limit_per_type,
|
| 81 |
+
"top_limit": self.top_limit,
|
| 82 |
+
"as_of": self.as_of.astimezone(UTC).isoformat(),
|
| 83 |
+
"location": self.location.model_dump(mode="json") if self.location else None,
|
| 84 |
+
"filters": filters,
|
| 85 |
+
"nearby_boost": nearby_boost,
|
| 86 |
+
"policy_version": policy_version,
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class InternalSearchCandidateSchema(BaseModel):
|
| 91 |
+
id: str
|
| 92 |
+
resource_type: SearchResourceType
|
| 93 |
+
score: float
|
| 94 |
+
semantic_score: float | None = None
|
| 95 |
+
lexical_score: float | None = None
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class InternalSearchPaginationSchema(BaseModel):
|
| 99 |
+
returned_count: int
|
| 100 |
+
has_more: bool
|
| 101 |
+
next_cursor: str | None = None
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class InternalSearchMetadataSchema(BaseModel):
|
| 105 |
+
strategy: str
|
| 106 |
+
failed_resources: dict[str, str]
|
| 107 |
+
threshold_policy_version: str
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
class InternalSearchCandidatesResponse(BaseModel):
|
| 111 |
+
query: str
|
| 112 |
+
sections: dict[str, list[InternalSearchCandidateSchema]]
|
| 113 |
+
pagination: dict[str, InternalSearchPaginationSchema]
|
| 114 |
+
metadata: InternalSearchMetadataSchema
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def candidates_result_to_schema(
|
| 118 |
+
result: SearchCandidatesResult,
|
| 119 |
+
) -> InternalSearchCandidatesResponse:
|
| 120 |
+
return InternalSearchCandidatesResponse(
|
| 121 |
+
query=result.query,
|
| 122 |
+
sections={
|
| 123 |
+
resource_type.value: [_candidate_to_schema(item) for item in items]
|
| 124 |
+
for resource_type, items in result.sections.items()
|
| 125 |
+
},
|
| 126 |
+
pagination={
|
| 127 |
+
resource_type.value: InternalSearchPaginationSchema(
|
| 128 |
+
returned_count=page.returned_count,
|
| 129 |
+
has_more=(
|
| 130 |
+
page.has_more
|
| 131 |
+
and page.next_offset is not None
|
| 132 |
+
and page.next_offset <= MAX_SEARCH_OFFSET
|
| 133 |
+
),
|
| 134 |
+
next_cursor=(
|
| 135 |
+
encode_search_cursor(
|
| 136 |
+
resource_type,
|
| 137 |
+
result.cursor_context,
|
| 138 |
+
page.next_offset,
|
| 139 |
+
)
|
| 140 |
+
if page.next_offset is not None
|
| 141 |
+
and page.next_offset <= MAX_SEARCH_OFFSET
|
| 142 |
+
else None
|
| 143 |
+
),
|
| 144 |
+
)
|
| 145 |
+
for resource_type, page in result.pagination.items()
|
| 146 |
+
},
|
| 147 |
+
metadata=InternalSearchMetadataSchema(
|
| 148 |
+
strategy="hybrid_thresholded_candidates_v2",
|
| 149 |
+
failed_resources={
|
| 150 |
+
resource_type.value: error
|
| 151 |
+
for resource_type, error in result.failed_resources.items()
|
| 152 |
+
},
|
| 153 |
+
threshold_policy_version=result.policy_version,
|
| 154 |
+
),
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _candidate_to_schema(item: SearchCandidate) -> InternalSearchCandidateSchema:
|
| 159 |
+
return InternalSearchCandidateSchema(
|
| 160 |
+
id=item.id,
|
| 161 |
+
resource_type=item.resource_type,
|
| 162 |
+
score=round(item.score, 6),
|
| 163 |
+
semantic_score=(
|
| 164 |
+
round(item.semantic_score, 6) if item.semantic_score is not None else None
|
| 165 |
+
),
|
| 166 |
+
lexical_score=(
|
| 167 |
+
round(item.lexical_score, 6) if item.lexical_score is not None else None
|
| 168 |
+
),
|
| 169 |
+
)
|
app/modules/search/api/public_router.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import secrets
|
| 2 |
+
|
| 3 |
+
from fastapi import APIRouter, Depends, Header, HTTPException, status
|
| 4 |
+
|
| 5 |
+
from app.modules.search.api.cursor import (
|
| 6 |
+
build_search_cursor_context,
|
| 7 |
+
decode_search_cursor,
|
| 8 |
+
)
|
| 9 |
+
from app.modules.search.api.dependencies import get_search_all_use_case
|
| 10 |
+
from app.modules.search.api.schemas import (
|
| 11 |
+
GlobalSearchRequest,
|
| 12 |
+
GlobalSearchResponse,
|
| 13 |
+
result_to_schema,
|
| 14 |
+
)
|
| 15 |
+
from app.modules.search.application.use_cases.search_all import SearchAllUseCase
|
| 16 |
+
from app.modules.search.domain.models import ALL_SEARCH_RESOURCE_TYPES
|
| 17 |
+
from app.shared.config.settings import get_settings
|
| 18 |
+
from app.shared.security.rate_limit import rate_limit_placeholder
|
| 19 |
+
|
| 20 |
+
router = APIRouter(
|
| 21 |
+
tags=["search"],
|
| 22 |
+
dependencies=[Depends(rate_limit_placeholder)],
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@router.post("/search", response_model=GlobalSearchResponse)
|
| 27 |
+
async def search_all(
|
| 28 |
+
payload: GlobalSearchRequest,
|
| 29 |
+
authorization: str | None = Header(default=None, alias="Authorization"),
|
| 30 |
+
use_case: SearchAllUseCase = Depends(get_search_all_use_case),
|
| 31 |
+
) -> GlobalSearchResponse:
|
| 32 |
+
settings = get_settings()
|
| 33 |
+
if not settings.public_global_search_enabled:
|
| 34 |
+
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
| 35 |
+
if payload.requester_id:
|
| 36 |
+
expected_token = settings.search_internal_token
|
| 37 |
+
bearer_token = _extract_bearer_token(authorization)
|
| 38 |
+
if not expected_token or not bearer_token or not secrets.compare_digest(
|
| 39 |
+
bearer_token, expected_token
|
| 40 |
+
):
|
| 41 |
+
raise HTTPException(
|
| 42 |
+
status_code=403,
|
| 43 |
+
detail="A trusted internal token is required for requester-scoped search",
|
| 44 |
+
)
|
| 45 |
+
resource_types = (
|
| 46 |
+
tuple(dict.fromkeys(payload.resource_types))
|
| 47 |
+
if payload.resource_types
|
| 48 |
+
else ALL_SEARCH_RESOURCE_TYPES
|
| 49 |
+
)
|
| 50 |
+
cursor_context = build_search_cursor_context(
|
| 51 |
+
payload.cursor_context_payload(
|
| 52 |
+
settings.global_search_nearby_boost,
|
| 53 |
+
settings.global_search_threshold_policy_version,
|
| 54 |
+
)
|
| 55 |
+
)
|
| 56 |
+
try:
|
| 57 |
+
offsets = {
|
| 58 |
+
resource_type: decode_search_cursor(
|
| 59 |
+
cursor=cursor,
|
| 60 |
+
resource_type=resource_type,
|
| 61 |
+
context_fingerprint=cursor_context,
|
| 62 |
+
)
|
| 63 |
+
for resource_type, cursor in payload.cursors.items()
|
| 64 |
+
}
|
| 65 |
+
except ValueError as exc:
|
| 66 |
+
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
| 67 |
+
result = await use_case.execute(
|
| 68 |
+
query=payload.query,
|
| 69 |
+
resource_types=resource_types,
|
| 70 |
+
per_type_limit=payload.per_type_limit,
|
| 71 |
+
top_limit=payload.top_limit,
|
| 72 |
+
requester_id=str(payload.requester_id) if payload.requester_id else None,
|
| 73 |
+
offsets=offsets,
|
| 74 |
+
criteria=payload.to_domain_criteria(settings.global_search_nearby_boost),
|
| 75 |
+
cursor_context=cursor_context,
|
| 76 |
+
)
|
| 77 |
+
return result_to_schema(result)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _extract_bearer_token(authorization: str | None) -> str | None:
|
| 81 |
+
if not authorization:
|
| 82 |
+
return None
|
| 83 |
+
scheme, separator, token = authorization.partition(" ")
|
| 84 |
+
if not separator or scheme.lower() != "bearer" or not token.strip():
|
| 85 |
+
return None
|
| 86 |
+
return token.strip()
|
app/modules/search/api/router.py
CHANGED
|
@@ -1,81 +1,8 @@
|
|
| 1 |
-
import
|
| 2 |
|
| 3 |
-
from
|
|
|
|
| 4 |
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
decode_search_cursor,
|
| 9 |
-
)
|
| 10 |
-
from app.modules.search.api.schemas import (
|
| 11 |
-
GlobalSearchRequest,
|
| 12 |
-
GlobalSearchResponse,
|
| 13 |
-
result_to_schema,
|
| 14 |
-
)
|
| 15 |
-
from app.modules.search.application.use_cases.search_all import SearchAllUseCase
|
| 16 |
-
from app.modules.search.domain.models import ALL_SEARCH_RESOURCE_TYPES
|
| 17 |
-
from app.shared.config.settings import get_settings
|
| 18 |
-
from app.shared.security.rate_limit import rate_limit_placeholder
|
| 19 |
-
|
| 20 |
-
router = APIRouter(
|
| 21 |
-
tags=["search"],
|
| 22 |
-
dependencies=[Depends(rate_limit_placeholder)],
|
| 23 |
-
)
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
@router.post("/search", response_model=GlobalSearchResponse)
|
| 27 |
-
async def search_all(
|
| 28 |
-
payload: GlobalSearchRequest,
|
| 29 |
-
authorization: str | None = Header(default=None, alias="Authorization"),
|
| 30 |
-
use_case: SearchAllUseCase = Depends(get_search_all_use_case),
|
| 31 |
-
) -> GlobalSearchResponse:
|
| 32 |
-
settings = get_settings()
|
| 33 |
-
if payload.requester_id:
|
| 34 |
-
expected_token = settings.search_internal_token
|
| 35 |
-
bearer_token = _extract_bearer_token(authorization)
|
| 36 |
-
if not expected_token or not bearer_token or not secrets.compare_digest(
|
| 37 |
-
bearer_token, expected_token
|
| 38 |
-
):
|
| 39 |
-
raise HTTPException(
|
| 40 |
-
status_code=403,
|
| 41 |
-
detail="A trusted internal token is required for requester-scoped search",
|
| 42 |
-
)
|
| 43 |
-
resource_types = (
|
| 44 |
-
tuple(dict.fromkeys(payload.resource_types))
|
| 45 |
-
if payload.resource_types
|
| 46 |
-
else ALL_SEARCH_RESOURCE_TYPES
|
| 47 |
-
)
|
| 48 |
-
cursor_context = build_search_cursor_context(
|
| 49 |
-
payload.cursor_context_payload(settings.global_search_nearby_boost)
|
| 50 |
-
)
|
| 51 |
-
try:
|
| 52 |
-
offsets = {
|
| 53 |
-
resource_type: decode_search_cursor(
|
| 54 |
-
cursor=cursor,
|
| 55 |
-
resource_type=resource_type,
|
| 56 |
-
context_fingerprint=cursor_context,
|
| 57 |
-
)
|
| 58 |
-
for resource_type, cursor in payload.cursors.items()
|
| 59 |
-
}
|
| 60 |
-
except ValueError as exc:
|
| 61 |
-
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
| 62 |
-
result = await use_case.execute(
|
| 63 |
-
query=payload.query,
|
| 64 |
-
resource_types=resource_types,
|
| 65 |
-
per_type_limit=payload.per_type_limit,
|
| 66 |
-
top_limit=payload.top_limit,
|
| 67 |
-
requester_id=str(payload.requester_id) if payload.requester_id else None,
|
| 68 |
-
offsets=offsets,
|
| 69 |
-
criteria=payload.to_domain_criteria(settings.global_search_nearby_boost),
|
| 70 |
-
cursor_context=cursor_context,
|
| 71 |
-
)
|
| 72 |
-
return result_to_schema(result)
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
def _extract_bearer_token(authorization: str | None) -> str | None:
|
| 76 |
-
if not authorization:
|
| 77 |
-
return None
|
| 78 |
-
scheme, separator, token = authorization.partition(" ")
|
| 79 |
-
if not separator or scheme.lower() != "bearer" or not token.strip():
|
| 80 |
-
return None
|
| 81 |
-
return token.strip()
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
|
| 3 |
+
from app.modules.search.api.internal_router import router as internal_router
|
| 4 |
+
from app.modules.search.api.public_router import router as public_router
|
| 5 |
|
| 6 |
+
router = APIRouter()
|
| 7 |
+
router.include_router(public_router)
|
| 8 |
+
router.include_router(internal_router)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/modules/search/api/schemas.py
CHANGED
|
@@ -13,7 +13,7 @@ from app.modules.search.domain.filters import (
|
|
| 13 |
SearchLocation,
|
| 14 |
)
|
| 15 |
from app.modules.search.domain.models import SearchAllResult, SearchHit, SearchResourceType
|
| 16 |
-
from app.
|
| 17 |
|
| 18 |
|
| 19 |
class SearchLocationSchema(BaseModel):
|
|
@@ -121,7 +121,11 @@ class GlobalSearchRequest(BaseModel):
|
|
| 121 |
nearby_boost=nearby_boost,
|
| 122 |
)
|
| 123 |
|
| 124 |
-
def cursor_context_payload(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
filters = self.filters.model_dump(mode="json")
|
| 126 |
for key in ("categories", "price_ranges", "tags", "user_roles"):
|
| 127 |
filters[key] = sorted(
|
|
@@ -140,7 +144,7 @@ class GlobalSearchRequest(BaseModel):
|
|
| 140 |
value.astimezone(UTC).isoformat() if value is not None else None
|
| 141 |
)
|
| 142 |
return {
|
| 143 |
-
"query":
|
| 144 |
"per_type_limit": self.per_type_limit,
|
| 145 |
"requester_id": str(self.requester_id) if self.requester_id else None,
|
| 146 |
"location": (
|
|
@@ -148,6 +152,7 @@ class GlobalSearchRequest(BaseModel):
|
|
| 148 |
),
|
| 149 |
"filters": filters,
|
| 150 |
"nearby_boost": nearby_boost,
|
|
|
|
| 151 |
}
|
| 152 |
|
| 153 |
|
|
|
|
| 13 |
SearchLocation,
|
| 14 |
)
|
| 15 |
from app.modules.search.domain.models import SearchAllResult, SearchHit, SearchResourceType
|
| 16 |
+
from app.modules.search.domain.query import normalize_search_query
|
| 17 |
|
| 18 |
|
| 19 |
class SearchLocationSchema(BaseModel):
|
|
|
|
| 121 |
nearby_boost=nearby_boost,
|
| 122 |
)
|
| 123 |
|
| 124 |
+
def cursor_context_payload(
|
| 125 |
+
self,
|
| 126 |
+
nearby_boost: float = 0.12,
|
| 127 |
+
policy_version: str = "global-search-relevance-v1",
|
| 128 |
+
) -> dict[str, Any]:
|
| 129 |
filters = self.filters.model_dump(mode="json")
|
| 130 |
for key in ("categories", "price_ranges", "tags", "user_roles"):
|
| 131 |
filters[key] = sorted(
|
|
|
|
| 144 |
value.astimezone(UTC).isoformat() if value is not None else None
|
| 145 |
)
|
| 146 |
return {
|
| 147 |
+
"query": normalize_search_query(self.query),
|
| 148 |
"per_type_limit": self.per_type_limit,
|
| 149 |
"requester_id": str(self.requester_id) if self.requester_id else None,
|
| 150 |
"location": (
|
|
|
|
| 152 |
),
|
| 153 |
"filters": filters,
|
| 154 |
"nearby_boost": nearby_boost,
|
| 155 |
+
"policy_version": policy_version,
|
| 156 |
}
|
| 157 |
|
| 158 |
|
app/modules/search/application/ports/query_embedding_provider.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Protocol
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class QueryEmbeddingProvider(Protocol):
|
| 5 |
+
def embed_text(self, text: str) -> list[float]:
|
| 6 |
+
"""Build one embedding for a normalized search query."""
|
app/modules/search/application/use_cases/search_all.py
CHANGED
|
@@ -1,8 +1,13 @@
|
|
| 1 |
import asyncio
|
|
|
|
| 2 |
import math
|
|
|
|
| 3 |
|
| 4 |
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 5 |
from app.modules.search.application.ports.nearby_place_provider import NearbyPlaceProvider
|
|
|
|
|
|
|
|
|
|
| 6 |
from app.modules.search.domain.filters import SearchCriteria
|
| 7 |
from app.modules.search.domain.models import (
|
| 8 |
ALL_SEARCH_RESOURCE_TYPES,
|
|
@@ -11,17 +16,15 @@ from app.modules.search.domain.models import (
|
|
| 11 |
SearchResourceType,
|
| 12 |
SearchSectionPagination,
|
| 13 |
)
|
| 14 |
-
from app.
|
| 15 |
-
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 16 |
-
from app.shared.nlp.preprocessing.text import prepare_for_embedding
|
| 17 |
|
| 18 |
-
logger =
|
| 19 |
|
| 20 |
|
| 21 |
class SearchAllUseCase:
|
| 22 |
def __init__(
|
| 23 |
self,
|
| 24 |
-
embedding_provider:
|
| 25 |
providers: list[SearchProvider],
|
| 26 |
nearby_place_provider: NearbyPlaceProvider | None = None,
|
| 27 |
) -> None:
|
|
@@ -40,10 +43,14 @@ class SearchAllUseCase:
|
|
| 40 |
criteria: SearchCriteria | None = None,
|
| 41 |
cursor_context: str = "",
|
| 42 |
) -> SearchAllResult:
|
| 43 |
-
normalized_query =
|
| 44 |
embedding = self._embedding_provider.embed_text(normalized_query)
|
| 45 |
effective_offsets = offsets or {}
|
| 46 |
effective_criteria = criteria or SearchCriteria()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
if effective_criteria.location is not None:
|
| 48 |
if self._nearby_place_provider is None:
|
| 49 |
raise RuntimeError("nearby place provider is not configured")
|
|
|
|
| 1 |
import asyncio
|
| 2 |
+
import logging
|
| 3 |
import math
|
| 4 |
+
from datetime import UTC, datetime
|
| 5 |
|
| 6 |
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 7 |
from app.modules.search.application.ports.nearby_place_provider import NearbyPlaceProvider
|
| 8 |
+
from app.modules.search.application.ports.query_embedding_provider import (
|
| 9 |
+
QueryEmbeddingProvider,
|
| 10 |
+
)
|
| 11 |
from app.modules.search.domain.filters import SearchCriteria
|
| 12 |
from app.modules.search.domain.models import (
|
| 13 |
ALL_SEARCH_RESOURCE_TYPES,
|
|
|
|
| 16 |
SearchResourceType,
|
| 17 |
SearchSectionPagination,
|
| 18 |
)
|
| 19 |
+
from app.modules.search.domain.query import normalize_search_query
|
|
|
|
|
|
|
| 20 |
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
|
| 23 |
|
| 24 |
class SearchAllUseCase:
|
| 25 |
def __init__(
|
| 26 |
self,
|
| 27 |
+
embedding_provider: QueryEmbeddingProvider,
|
| 28 |
providers: list[SearchProvider],
|
| 29 |
nearby_place_provider: NearbyPlaceProvider | None = None,
|
| 30 |
) -> None:
|
|
|
|
| 43 |
criteria: SearchCriteria | None = None,
|
| 44 |
cursor_context: str = "",
|
| 45 |
) -> SearchAllResult:
|
| 46 |
+
normalized_query = normalize_search_query(query)
|
| 47 |
embedding = self._embedding_provider.embed_text(normalized_query)
|
| 48 |
effective_offsets = offsets or {}
|
| 49 |
effective_criteria = criteria or SearchCriteria()
|
| 50 |
+
if effective_criteria.event_active_at is None:
|
| 51 |
+
effective_criteria = effective_criteria.with_event_active_at(
|
| 52 |
+
datetime.now(UTC)
|
| 53 |
+
)
|
| 54 |
if effective_criteria.location is not None:
|
| 55 |
if self._nearby_place_provider is None:
|
| 56 |
raise RuntimeError("nearby place provider is not configured")
|
app/modules/search/application/use_cases/search_candidates.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import logging
|
| 3 |
+
from datetime import datetime
|
| 4 |
+
|
| 5 |
+
from app.modules.search.application.ports.nearby_place_provider import (
|
| 6 |
+
NearbyPlaceProvider,
|
| 7 |
+
)
|
| 8 |
+
from app.modules.search.application.ports.query_embedding_provider import (
|
| 9 |
+
QueryEmbeddingProvider,
|
| 10 |
+
)
|
| 11 |
+
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 12 |
+
from app.modules.search.domain.filters import SearchCriteria
|
| 13 |
+
from app.modules.search.domain.models import (
|
| 14 |
+
SearchCandidate,
|
| 15 |
+
SearchCandidatesResult,
|
| 16 |
+
SearchResourceType,
|
| 17 |
+
SearchSectionPagination,
|
| 18 |
+
)
|
| 19 |
+
from app.modules.search.domain.query import normalize_search_query
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class SearchCandidatesUseCase:
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
embedding_provider: QueryEmbeddingProvider,
|
| 28 |
+
providers: list[SearchProvider],
|
| 29 |
+
policy_version: str,
|
| 30 |
+
nearby_place_provider: NearbyPlaceProvider | None = None,
|
| 31 |
+
) -> None:
|
| 32 |
+
self._embedding_provider = embedding_provider
|
| 33 |
+
self._providers = {provider.resource_type: provider for provider in providers}
|
| 34 |
+
self._nearby_place_provider = nearby_place_provider
|
| 35 |
+
self._policy_version = policy_version
|
| 36 |
+
|
| 37 |
+
async def execute(
|
| 38 |
+
self,
|
| 39 |
+
query: str,
|
| 40 |
+
resource_types: tuple[SearchResourceType, ...],
|
| 41 |
+
candidate_limit_per_type: int,
|
| 42 |
+
as_of: datetime,
|
| 43 |
+
offsets: dict[SearchResourceType, int] | None = None,
|
| 44 |
+
criteria: SearchCriteria | None = None,
|
| 45 |
+
cursor_context: str = "",
|
| 46 |
+
) -> SearchCandidatesResult:
|
| 47 |
+
normalized_query = normalize_search_query(query)
|
| 48 |
+
embedding = self._embedding_provider.embed_text(normalized_query)
|
| 49 |
+
effective_offsets = offsets or {}
|
| 50 |
+
effective_criteria = (criteria or SearchCriteria()).with_event_active_at(as_of)
|
| 51 |
+
if effective_criteria.location is not None:
|
| 52 |
+
if self._nearby_place_provider is None:
|
| 53 |
+
raise RuntimeError("nearby place provider is not configured")
|
| 54 |
+
nearby_ids = await self._nearby_place_provider.get_nearby_place_ids(
|
| 55 |
+
latitude=effective_criteria.location.latitude,
|
| 56 |
+
longitude=effective_criteria.location.longitude,
|
| 57 |
+
radius_meters=effective_criteria.location.radius_meters,
|
| 58 |
+
)
|
| 59 |
+
effective_criteria = effective_criteria.with_nearby_place_ids(nearby_ids)
|
| 60 |
+
|
| 61 |
+
providers = [
|
| 62 |
+
self._providers[resource_type]
|
| 63 |
+
for resource_type in resource_types
|
| 64 |
+
if resource_type in self._providers
|
| 65 |
+
]
|
| 66 |
+
outcomes = await asyncio.gather(
|
| 67 |
+
*[
|
| 68 |
+
provider.search(
|
| 69 |
+
query=normalized_query,
|
| 70 |
+
embedding=embedding,
|
| 71 |
+
limit=candidate_limit_per_type + 1,
|
| 72 |
+
offset=effective_offsets.get(provider.resource_type, 0),
|
| 73 |
+
requester_id=None,
|
| 74 |
+
criteria=effective_criteria,
|
| 75 |
+
)
|
| 76 |
+
for provider in providers
|
| 77 |
+
],
|
| 78 |
+
return_exceptions=True,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
sections: dict[SearchResourceType, list[SearchCandidate]] = {
|
| 82 |
+
resource_type: [] for resource_type in resource_types
|
| 83 |
+
}
|
| 84 |
+
pagination: dict[SearchResourceType, SearchSectionPagination] = {}
|
| 85 |
+
failures: dict[SearchResourceType, str] = {}
|
| 86 |
+
for provider, outcome in zip(providers, outcomes):
|
| 87 |
+
if isinstance(outcome, BaseException):
|
| 88 |
+
logger.error(
|
| 89 |
+
"Internal search provider failed resource=%s error=%s",
|
| 90 |
+
provider.resource_type,
|
| 91 |
+
type(outcome).__name__,
|
| 92 |
+
)
|
| 93 |
+
failures[provider.resource_type] = type(outcome).__name__
|
| 94 |
+
pagination[provider.resource_type] = SearchSectionPagination(
|
| 95 |
+
page_size=candidate_limit_per_type,
|
| 96 |
+
returned_count=0,
|
| 97 |
+
has_more=False,
|
| 98 |
+
)
|
| 99 |
+
continue
|
| 100 |
+
ranked = list(outcome)
|
| 101 |
+
page_hits = ranked[:candidate_limit_per_type]
|
| 102 |
+
has_more = len(ranked) > candidate_limit_per_type
|
| 103 |
+
current_offset = effective_offsets.get(provider.resource_type, 0)
|
| 104 |
+
sections[provider.resource_type] = [
|
| 105 |
+
SearchCandidate(
|
| 106 |
+
id=hit.id,
|
| 107 |
+
resource_type=hit.resource_type,
|
| 108 |
+
score=hit.score,
|
| 109 |
+
semantic_score=hit.semantic_score,
|
| 110 |
+
lexical_score=hit.lexical_score,
|
| 111 |
+
)
|
| 112 |
+
for hit in page_hits
|
| 113 |
+
]
|
| 114 |
+
pagination[provider.resource_type] = SearchSectionPagination(
|
| 115 |
+
page_size=candidate_limit_per_type,
|
| 116 |
+
returned_count=len(page_hits),
|
| 117 |
+
has_more=has_more,
|
| 118 |
+
next_offset=(current_offset + len(page_hits) if has_more else None),
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
for resource_type in resource_types:
|
| 122 |
+
pagination.setdefault(
|
| 123 |
+
resource_type,
|
| 124 |
+
SearchSectionPagination(
|
| 125 |
+
page_size=candidate_limit_per_type,
|
| 126 |
+
returned_count=0,
|
| 127 |
+
has_more=False,
|
| 128 |
+
),
|
| 129 |
+
)
|
| 130 |
+
return SearchCandidatesResult(
|
| 131 |
+
query=query,
|
| 132 |
+
normalized_query=normalized_query,
|
| 133 |
+
sections=sections,
|
| 134 |
+
pagination=pagination,
|
| 135 |
+
failed_resources=failures,
|
| 136 |
+
cursor_context=cursor_context,
|
| 137 |
+
policy_version=self._policy_version,
|
| 138 |
+
)
|
app/modules/search/domain/filters.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
from dataclasses import dataclass, field, replace
|
| 2 |
-
from datetime import UTC, datetime
|
| 3 |
from enum import StrEnum
|
| 4 |
from typing import Any, Sequence
|
| 5 |
|
|
@@ -62,10 +62,14 @@ class SearchCriteria:
|
|
| 62 |
location: SearchLocation | None = None
|
| 63 |
nearby_place_ids: frozenset[str] = frozenset()
|
| 64 |
nearby_boost: float = 0.12
|
|
|
|
| 65 |
|
| 66 |
def with_nearby_place_ids(self, place_ids: set[str]) -> "SearchCriteria":
|
| 67 |
return replace(self, nearby_place_ids=frozenset(place_ids))
|
| 68 |
|
|
|
|
|
|
|
|
|
|
| 69 |
def requires_extended_candidates(self, resource_type: SearchResourceType) -> bool:
|
| 70 |
if not self.filters.is_empty():
|
| 71 |
return True
|
|
@@ -86,6 +90,12 @@ def filter_and_rank_hits(
|
|
| 86 |
for hit in hits:
|
| 87 |
if not _matches_filters(resource_type, hit.metadata, criteria.filters):
|
| 88 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
is_nearby = _is_nearby(resource_type, hit, criteria.nearby_place_ids)
|
| 90 |
location_applicable = resource_type in _LOCATION_AWARE_RESOURCES
|
| 91 |
if (
|
|
@@ -178,6 +188,22 @@ def _matches_filters(
|
|
| 178 |
return True
|
| 179 |
|
| 180 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
def _is_nearby(
|
| 182 |
resource_type: SearchResourceType,
|
| 183 |
hit: SearchHit,
|
|
|
|
| 1 |
from dataclasses import dataclass, field, replace
|
| 2 |
+
from datetime import UTC, datetime, timedelta
|
| 3 |
from enum import StrEnum
|
| 4 |
from typing import Any, Sequence
|
| 5 |
|
|
|
|
| 62 |
location: SearchLocation | None = None
|
| 63 |
nearby_place_ids: frozenset[str] = frozenset()
|
| 64 |
nearby_boost: float = 0.12
|
| 65 |
+
event_active_at: datetime | None = None
|
| 66 |
|
| 67 |
def with_nearby_place_ids(self, place_ids: set[str]) -> "SearchCriteria":
|
| 68 |
return replace(self, nearby_place_ids=frozenset(place_ids))
|
| 69 |
|
| 70 |
+
def with_event_active_at(self, value: datetime) -> "SearchCriteria":
|
| 71 |
+
return replace(self, event_active_at=value)
|
| 72 |
+
|
| 73 |
def requires_extended_candidates(self, resource_type: SearchResourceType) -> bool:
|
| 74 |
if not self.filters.is_empty():
|
| 75 |
return True
|
|
|
|
| 90 |
for hit in hits:
|
| 91 |
if not _matches_filters(resource_type, hit.metadata, criteria.filters):
|
| 92 |
continue
|
| 93 |
+
if (
|
| 94 |
+
resource_type == SearchResourceType.EVENTS
|
| 95 |
+
and criteria.event_active_at is not None
|
| 96 |
+
and not _event_is_active(hit.metadata, criteria.event_active_at)
|
| 97 |
+
):
|
| 98 |
+
continue
|
| 99 |
is_nearby = _is_nearby(resource_type, hit, criteria.nearby_place_ids)
|
| 100 |
location_applicable = resource_type in _LOCATION_AWARE_RESOURCES
|
| 101 |
if (
|
|
|
|
| 188 |
return True
|
| 189 |
|
| 190 |
|
| 191 |
+
def _event_is_active(metadata: dict[str, Any], active_at: datetime) -> bool:
|
| 192 |
+
start_time = _parse_datetime(metadata.get("start_time"))
|
| 193 |
+
if start_time is None:
|
| 194 |
+
return False
|
| 195 |
+
raw_duration = metadata.get("duration_minutes")
|
| 196 |
+
if isinstance(raw_duration, bool):
|
| 197 |
+
return False
|
| 198 |
+
try:
|
| 199 |
+
duration_minutes = int(raw_duration)
|
| 200 |
+
except (TypeError, ValueError):
|
| 201 |
+
return False
|
| 202 |
+
if duration_minutes <= 0:
|
| 203 |
+
return False
|
| 204 |
+
return start_time + timedelta(minutes=duration_minutes) > _normalized_datetime(active_at)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
def _is_nearby(
|
| 208 |
resource_type: SearchResourceType,
|
| 209 |
hit: SearchHit,
|
app/modules/search/domain/models.py
CHANGED
|
@@ -38,6 +38,26 @@ class SearchSectionPagination:
|
|
| 38 |
next_offset: int | None = None
|
| 39 |
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
@dataclass(frozen=True)
|
| 42 |
class SearchAllResult:
|
| 43 |
query: str
|
|
|
|
| 38 |
next_offset: int | None = None
|
| 39 |
|
| 40 |
|
| 41 |
+
@dataclass(frozen=True)
|
| 42 |
+
class SearchCandidate:
|
| 43 |
+
id: str
|
| 44 |
+
resource_type: SearchResourceType
|
| 45 |
+
score: float
|
| 46 |
+
semantic_score: float | None = None
|
| 47 |
+
lexical_score: float | None = None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@dataclass(frozen=True)
|
| 51 |
+
class SearchCandidatesResult:
|
| 52 |
+
query: str
|
| 53 |
+
normalized_query: str
|
| 54 |
+
sections: dict[SearchResourceType, list[SearchCandidate]]
|
| 55 |
+
pagination: dict[SearchResourceType, SearchSectionPagination]
|
| 56 |
+
failed_resources: dict[SearchResourceType, str]
|
| 57 |
+
cursor_context: str
|
| 58 |
+
policy_version: str
|
| 59 |
+
|
| 60 |
+
|
| 61 |
@dataclass(frozen=True)
|
| 62 |
class SearchAllResult:
|
| 63 |
query: str
|
app/modules/search/domain/query.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import unicodedata
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def normalize_search_query(value: str) -> str:
|
| 6 |
+
normalized = unicodedata.normalize("NFKD", (value or "").casefold())
|
| 7 |
+
without_accents = "".join(
|
| 8 |
+
character
|
| 9 |
+
for character in normalized
|
| 10 |
+
if not unicodedata.combining(character)
|
| 11 |
+
)
|
| 12 |
+
return re.sub(r"\s+", " ", without_accents).strip()
|
app/modules/search/domain/relevance.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
|
| 4 |
+
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(frozen=True)
|
| 8 |
+
class SearchRelevanceThreshold:
|
| 9 |
+
semantic_min: float
|
| 10 |
+
lexical_min: float
|
| 11 |
+
|
| 12 |
+
def __post_init__(self) -> None:
|
| 13 |
+
if not math.isfinite(self.semantic_min) or not -1.0 <= self.semantic_min <= 1.0:
|
| 14 |
+
raise ValueError("semantic_min must be finite and between -1 and 1")
|
| 15 |
+
if not math.isfinite(self.lexical_min) or self.lexical_min < 0.0:
|
| 16 |
+
raise ValueError("lexical_min must be finite and non-negative")
|
| 17 |
+
|
| 18 |
+
def accepts(self, hit: SearchHit) -> bool:
|
| 19 |
+
semantic_match = (
|
| 20 |
+
hit.semantic_score is not None
|
| 21 |
+
and math.isfinite(hit.semantic_score)
|
| 22 |
+
and hit.semantic_score >= self.semantic_min
|
| 23 |
+
)
|
| 24 |
+
lexical_match = (
|
| 25 |
+
hit.lexical_score is not None
|
| 26 |
+
and math.isfinite(hit.lexical_score)
|
| 27 |
+
and hit.lexical_score >= self.lexical_min
|
| 28 |
+
)
|
| 29 |
+
return semantic_match or lexical_match
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass(frozen=True)
|
| 33 |
+
class SearchRelevancePolicy:
|
| 34 |
+
thresholds: dict[SearchResourceType, SearchRelevanceThreshold]
|
| 35 |
+
version: str
|
| 36 |
+
|
| 37 |
+
def __post_init__(self) -> None:
|
| 38 |
+
missing = set(SearchResourceType) - set(self.thresholds)
|
| 39 |
+
extra = set(self.thresholds) - set(SearchResourceType)
|
| 40 |
+
if missing or extra:
|
| 41 |
+
raise ValueError("relevance policy must define every search resource")
|
| 42 |
+
if not self.version.strip() or len(self.version) > 64:
|
| 43 |
+
raise ValueError("relevance policy version is invalid")
|
| 44 |
+
|
| 45 |
+
def threshold_for(
|
| 46 |
+
self, resource_type: SearchResourceType
|
| 47 |
+
) -> SearchRelevanceThreshold:
|
| 48 |
+
return self.thresholds[resource_type]
|
| 49 |
+
|
| 50 |
+
def accepts(self, hit: SearchHit) -> bool:
|
| 51 |
+
return self.threshold_for(hit.resource_type).accepts(hit)
|
| 52 |
+
|
| 53 |
+
@classmethod
|
| 54 |
+
def uniform(
|
| 55 |
+
cls,
|
| 56 |
+
semantic_min: float = 0.30,
|
| 57 |
+
lexical_min: float = 0.05,
|
| 58 |
+
version: str = "global-search-relevance-v1",
|
| 59 |
+
) -> "SearchRelevancePolicy":
|
| 60 |
+
return cls(
|
| 61 |
+
thresholds={
|
| 62 |
+
resource_type: SearchRelevanceThreshold(
|
| 63 |
+
semantic_min=semantic_min,
|
| 64 |
+
lexical_min=lexical_min,
|
| 65 |
+
)
|
| 66 |
+
for resource_type in SearchResourceType
|
| 67 |
+
},
|
| 68 |
+
version=version,
|
| 69 |
+
)
|
app/modules/search/infrastructure/mock_provider.py
CHANGED
|
@@ -7,6 +7,7 @@ from app.modules.posts.infrastructure.mock_post_repository import SAMPLE_POSTS
|
|
| 7 |
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 8 |
from app.modules.search.domain.filters import SearchCriteria, filter_and_rank_hits
|
| 9 |
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
|
|
|
| 10 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 11 |
from app.shared.nlp.preprocessing.text import prepare_for_embedding
|
| 12 |
|
|
@@ -25,8 +26,10 @@ class MockHybridSearchProvider(SearchProvider):
|
|
| 25 |
resource_type: SearchResourceType,
|
| 26 |
embedding_provider: EmbeddingProvider,
|
| 27 |
documents: list[MockSearchDocument],
|
|
|
|
| 28 |
) -> None:
|
| 29 |
self.resource_type = resource_type
|
|
|
|
| 30 |
self._records = [
|
| 31 |
(document, embedding_provider.embed_text(document.document))
|
| 32 |
for document in documents
|
|
@@ -70,7 +73,8 @@ class MockHybridSearchProvider(SearchProvider):
|
|
| 70 |
metadata=metadata,
|
| 71 |
)
|
| 72 |
)
|
| 73 |
-
|
|
|
|
| 74 |
return ranked[offset : offset + limit]
|
| 75 |
|
| 76 |
|
|
@@ -134,7 +138,11 @@ def get_mock_search_documents() -> dict[SearchResourceType, list[MockSearchDocum
|
|
| 134 |
id="event_birthday",
|
| 135 |
title="Fiesta de Cumpleanos",
|
| 136 |
document="fiesta cumpleanos evento independiente",
|
| 137 |
-
metadata={
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
)
|
| 139 |
],
|
| 140 |
}
|
|
|
|
| 7 |
from app.modules.search.application.ports.search_provider import SearchProvider
|
| 8 |
from app.modules.search.domain.filters import SearchCriteria, filter_and_rank_hits
|
| 9 |
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
| 10 |
+
from app.modules.search.domain.relevance import SearchRelevancePolicy
|
| 11 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 12 |
from app.shared.nlp.preprocessing.text import prepare_for_embedding
|
| 13 |
|
|
|
|
| 26 |
resource_type: SearchResourceType,
|
| 27 |
embedding_provider: EmbeddingProvider,
|
| 28 |
documents: list[MockSearchDocument],
|
| 29 |
+
relevance_policy: SearchRelevancePolicy | None = None,
|
| 30 |
) -> None:
|
| 31 |
self.resource_type = resource_type
|
| 32 |
+
self._relevance_policy = relevance_policy or SearchRelevancePolicy.uniform()
|
| 33 |
self._records = [
|
| 34 |
(document, embedding_provider.embed_text(document.document))
|
| 35 |
for document in documents
|
|
|
|
| 73 |
metadata=metadata,
|
| 74 |
)
|
| 75 |
)
|
| 76 |
+
relevant_hits = [hit for hit in hits if self._relevance_policy.accepts(hit)]
|
| 77 |
+
ranked = filter_and_rank_hits(self.resource_type, relevant_hits, criteria)
|
| 78 |
return ranked[offset : offset + limit]
|
| 79 |
|
| 80 |
|
|
|
|
| 138 |
id="event_birthday",
|
| 139 |
title="Fiesta de Cumpleanos",
|
| 140 |
document="fiesta cumpleanos evento independiente",
|
| 141 |
+
metadata={
|
| 142 |
+
"is_public": True,
|
| 143 |
+
"start_time": "2099-07-12T20:00:00Z",
|
| 144 |
+
"duration_minutes": 300,
|
| 145 |
+
},
|
| 146 |
)
|
| 147 |
],
|
| 148 |
}
|
app/modules/search/infrastructure/pgvector_provider.py
CHANGED
|
@@ -7,6 +7,7 @@ from app.modules.search.domain.filters import (
|
|
| 7 |
filter_and_rank_hits,
|
| 8 |
)
|
| 9 |
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
|
|
|
| 10 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
| 11 |
from app.shared.vector_store.models import VectorMatch
|
| 12 |
|
|
@@ -18,8 +19,13 @@ class PgvectorPlaceSearchProvider(SearchProvider):
|
|
| 18 |
|
| 19 |
resource_type = SearchResourceType.PLACES
|
| 20 |
|
| 21 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
self._vector_client = vector_client
|
|
|
|
| 23 |
|
| 24 |
async def search(
|
| 25 |
self,
|
|
@@ -48,7 +54,8 @@ class PgvectorPlaceSearchProvider(SearchProvider):
|
|
| 48 |
_to_search_hit(self.resource_type, match)
|
| 49 |
for match in matches
|
| 50 |
]
|
| 51 |
-
|
|
|
|
| 52 |
return ranked[offset : offset + limit]
|
| 53 |
|
| 54 |
|
|
@@ -57,9 +64,11 @@ class PgvectorHybridSearchProvider(SearchProvider):
|
|
| 57 |
self,
|
| 58 |
resource_type: SearchResourceType,
|
| 59 |
vector_client: AwsPgvectorClient,
|
|
|
|
| 60 |
) -> None:
|
| 61 |
self.resource_type = resource_type
|
| 62 |
self._vector_client = vector_client
|
|
|
|
| 63 |
|
| 64 |
async def search(
|
| 65 |
self,
|
|
@@ -78,6 +87,14 @@ class PgvectorHybridSearchProvider(SearchProvider):
|
|
| 78 |
):
|
| 79 |
return []
|
| 80 |
filters = {"is_active": True}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
if requester_id:
|
| 82 |
filters["requester_id"] = requester_id
|
| 83 |
filters.update(_hybrid_metadata_filters(criteria, self.resource_type))
|
|
@@ -93,7 +110,8 @@ class PgvectorHybridSearchProvider(SearchProvider):
|
|
| 93 |
_to_search_hit(self.resource_type, match)
|
| 94 |
for match in matches
|
| 95 |
]
|
| 96 |
-
|
|
|
|
| 97 |
return ranked[offset : offset + limit]
|
| 98 |
|
| 99 |
|
|
|
|
| 7 |
filter_and_rank_hits,
|
| 8 |
)
|
| 9 |
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
| 10 |
+
from app.modules.search.domain.relevance import SearchRelevancePolicy
|
| 11 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
| 12 |
from app.shared.vector_store.models import VectorMatch
|
| 13 |
|
|
|
|
| 19 |
|
| 20 |
resource_type = SearchResourceType.PLACES
|
| 21 |
|
| 22 |
+
def __init__(
|
| 23 |
+
self,
|
| 24 |
+
vector_client: AwsPgvectorClient,
|
| 25 |
+
relevance_policy: SearchRelevancePolicy | None = None,
|
| 26 |
+
) -> None:
|
| 27 |
self._vector_client = vector_client
|
| 28 |
+
self._relevance_policy = relevance_policy or SearchRelevancePolicy.uniform()
|
| 29 |
|
| 30 |
async def search(
|
| 31 |
self,
|
|
|
|
| 54 |
_to_search_hit(self.resource_type, match)
|
| 55 |
for match in matches
|
| 56 |
]
|
| 57 |
+
relevant_hits = [hit for hit in hits if self._relevance_policy.accepts(hit)]
|
| 58 |
+
ranked = filter_and_rank_hits(self.resource_type, relevant_hits, criteria)
|
| 59 |
return ranked[offset : offset + limit]
|
| 60 |
|
| 61 |
|
|
|
|
| 64 |
self,
|
| 65 |
resource_type: SearchResourceType,
|
| 66 |
vector_client: AwsPgvectorClient,
|
| 67 |
+
relevance_policy: SearchRelevancePolicy | None = None,
|
| 68 |
) -> None:
|
| 69 |
self.resource_type = resource_type
|
| 70 |
self._vector_client = vector_client
|
| 71 |
+
self._relevance_policy = relevance_policy or SearchRelevancePolicy.uniform()
|
| 72 |
|
| 73 |
async def search(
|
| 74 |
self,
|
|
|
|
| 87 |
):
|
| 88 |
return []
|
| 89 |
filters = {"is_active": True}
|
| 90 |
+
threshold = self._relevance_policy.threshold_for(self.resource_type)
|
| 91 |
+
filters["min_semantic_score"] = threshold.semantic_min
|
| 92 |
+
filters["min_lexical_score"] = threshold.lexical_min
|
| 93 |
+
if (
|
| 94 |
+
self.resource_type == SearchResourceType.EVENTS
|
| 95 |
+
and criteria.event_active_at is not None
|
| 96 |
+
):
|
| 97 |
+
filters["event_active_at"] = criteria.event_active_at.isoformat()
|
| 98 |
if requester_id:
|
| 99 |
filters["requester_id"] = requester_id
|
| 100 |
filters.update(_hybrid_metadata_filters(criteria, self.resource_type))
|
|
|
|
| 110 |
_to_search_hit(self.resource_type, match)
|
| 111 |
for match in matches
|
| 112 |
]
|
| 113 |
+
relevant_hits = [hit for hit in hits if self._relevance_policy.accepts(hit)]
|
| 114 |
+
ranked = filter_and_rank_hits(self.resource_type, relevant_hits, criteria)
|
| 115 |
return ranked[offset : offset + limit]
|
| 116 |
|
| 117 |
|
app/modules/search/infrastructure/relevance_policy.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from app.modules.search.domain.models import SearchResourceType
|
| 2 |
+
from app.modules.search.domain.relevance import (
|
| 3 |
+
SearchRelevancePolicy,
|
| 4 |
+
SearchRelevanceThreshold,
|
| 5 |
+
)
|
| 6 |
+
from app.shared.config.settings import Settings
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def build_relevance_policy(settings: Settings) -> SearchRelevancePolicy:
|
| 10 |
+
thresholds: dict[SearchResourceType, SearchRelevanceThreshold] = {}
|
| 11 |
+
for resource_type in SearchResourceType:
|
| 12 |
+
override = settings.global_search_resource_thresholds.get(
|
| 13 |
+
resource_type.value,
|
| 14 |
+
{},
|
| 15 |
+
)
|
| 16 |
+
thresholds[resource_type] = SearchRelevanceThreshold(
|
| 17 |
+
semantic_min=float(
|
| 18 |
+
override.get(
|
| 19 |
+
"semantic_min",
|
| 20 |
+
settings.global_search_min_semantic_score,
|
| 21 |
+
)
|
| 22 |
+
),
|
| 23 |
+
lexical_min=float(
|
| 24 |
+
override.get(
|
| 25 |
+
"lexical_min",
|
| 26 |
+
settings.global_search_min_lexical_score,
|
| 27 |
+
)
|
| 28 |
+
),
|
| 29 |
+
)
|
| 30 |
+
return SearchRelevancePolicy(
|
| 31 |
+
thresholds=thresholds,
|
| 32 |
+
version=settings.global_search_threshold_policy_version,
|
| 33 |
+
)
|
app/modules/users/infrastructure/main_api_user_source.py
CHANGED
|
@@ -13,7 +13,7 @@ class MainApiUsersClient(PagedMainApiSearchClient):
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
-
path=settings.
|
| 17 |
collection_keys=("users",),
|
| 18 |
mapper=user_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
|
|
|
| 13 |
def __init__(self, settings: Settings) -> None:
|
| 14 |
super().__init__(
|
| 15 |
settings=settings,
|
| 16 |
+
path=settings.main_api_users_snapshot_path,
|
| 17 |
collection_keys=("users",),
|
| 18 |
mapper=user_to_source_record,
|
| 19 |
page_limit=settings.main_api_search_page_limit,
|
app/shared/config/settings.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
from functools import lru_cache
|
|
|
|
| 2 |
|
| 3 |
from pydantic import AliasChoices, Field, model_validator
|
| 4 |
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
@@ -39,17 +40,21 @@ class Settings(BaseSettings):
|
|
| 39 |
default="/api/v1/internal/feed/interactions/changes",
|
| 40 |
alias="MAIN_API_FEED_INTERACTIONS_PATH",
|
| 41 |
)
|
| 42 |
-
|
| 43 |
-
default="/api/v1/
|
|
|
|
| 44 |
)
|
| 45 |
-
|
| 46 |
-
default="/api/v1/
|
|
|
|
| 47 |
)
|
| 48 |
-
|
| 49 |
-
default="/api/v1/
|
|
|
|
| 50 |
)
|
| 51 |
-
|
| 52 |
-
default="/api/v1/
|
|
|
|
| 53 |
)
|
| 54 |
main_api_internal_token: str | None = Field(
|
| 55 |
default=None,
|
|
@@ -85,6 +90,31 @@ class Settings(BaseSettings):
|
|
| 85 |
le=1.0,
|
| 86 |
alias="GLOBAL_SEARCH_NEARBY_BOOST",
|
| 87 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
groq_api_key: str | None = Field(default=None, alias="GROQ_API_KEY")
|
| 90 |
groq_model: str = Field(default="llama-3.1-8b-instant", alias="GROQ_MODEL")
|
|
@@ -154,7 +184,26 @@ class Settings(BaseSettings):
|
|
| 154 |
)
|
| 155 |
|
| 156 |
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
| 157 |
-
request_timeout_seconds: int = Field(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
llm_timeout_seconds: int = Field(default=12, alias="LLM_TIMEOUT_SECONDS")
|
| 159 |
max_llm_concurrent_requests: int = Field(
|
| 160 |
default=4,
|
|
@@ -194,6 +243,30 @@ class Settings(BaseSettings):
|
|
| 194 |
@model_validator(mode="after")
|
| 195 |
def validate_post_feed_security(self) -> "Settings":
|
| 196 |
self.vector_store_provider = self.vector_store_provider.strip().lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
if self.env.lower() != "local":
|
| 198 |
missing: list[str] = []
|
| 199 |
if not self.post_feed_internal_token:
|
|
|
|
| 1 |
from functools import lru_cache
|
| 2 |
+
import math
|
| 3 |
|
| 4 |
from pydantic import AliasChoices, Field, model_validator
|
| 5 |
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
| 40 |
default="/api/v1/internal/feed/interactions/changes",
|
| 41 |
alias="MAIN_API_FEED_INTERACTIONS_PATH",
|
| 42 |
)
|
| 43 |
+
main_api_users_snapshot_path: str = Field(
|
| 44 |
+
default="/api/v1/internal/search/users/snapshot",
|
| 45 |
+
alias="MAIN_API_USERS_SNAPSHOT_PATH",
|
| 46 |
)
|
| 47 |
+
main_api_clubs_snapshot_path: str = Field(
|
| 48 |
+
default="/api/v1/internal/search/clubs/snapshot",
|
| 49 |
+
alias="MAIN_API_CLUBS_SNAPSHOT_PATH",
|
| 50 |
)
|
| 51 |
+
main_api_groups_snapshot_path: str = Field(
|
| 52 |
+
default="/api/v1/internal/search/groups/snapshot",
|
| 53 |
+
alias="MAIN_API_GROUPS_SNAPSHOT_PATH",
|
| 54 |
)
|
| 55 |
+
main_api_events_snapshot_path: str = Field(
|
| 56 |
+
default="/api/v1/internal/search/events/snapshot",
|
| 57 |
+
alias="MAIN_API_EVENTS_SNAPSHOT_PATH",
|
| 58 |
)
|
| 59 |
main_api_internal_token: str | None = Field(
|
| 60 |
default=None,
|
|
|
|
| 90 |
le=1.0,
|
| 91 |
alias="GLOBAL_SEARCH_NEARBY_BOOST",
|
| 92 |
)
|
| 93 |
+
public_global_search_enabled: bool = Field(
|
| 94 |
+
default=True,
|
| 95 |
+
alias="PUBLIC_GLOBAL_SEARCH_ENABLED",
|
| 96 |
+
)
|
| 97 |
+
global_search_min_semantic_score: float = Field(
|
| 98 |
+
default=0.30,
|
| 99 |
+
ge=-1.0,
|
| 100 |
+
le=1.0,
|
| 101 |
+
alias="GLOBAL_SEARCH_MIN_SEMANTIC_SCORE",
|
| 102 |
+
)
|
| 103 |
+
global_search_min_lexical_score: float = Field(
|
| 104 |
+
default=0.05,
|
| 105 |
+
ge=0.0,
|
| 106 |
+
alias="GLOBAL_SEARCH_MIN_LEXICAL_SCORE",
|
| 107 |
+
)
|
| 108 |
+
global_search_resource_thresholds: dict[str, dict[str, float]] = Field(
|
| 109 |
+
default_factory=dict,
|
| 110 |
+
alias="GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON",
|
| 111 |
+
)
|
| 112 |
+
global_search_threshold_policy_version: str = Field(
|
| 113 |
+
default="global-search-relevance-v1",
|
| 114 |
+
min_length=1,
|
| 115 |
+
max_length=64,
|
| 116 |
+
alias="GLOBAL_SEARCH_THRESHOLD_POLICY_VERSION",
|
| 117 |
+
)
|
| 118 |
|
| 119 |
groq_api_key: str | None = Field(default=None, alias="GROQ_API_KEY")
|
| 120 |
groq_model: str = Field(default="llama-3.1-8b-instant", alias="GROQ_MODEL")
|
|
|
|
| 184 |
)
|
| 185 |
|
| 186 |
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
| 187 |
+
request_timeout_seconds: int = Field(
|
| 188 |
+
default=10,
|
| 189 |
+
gt=0,
|
| 190 |
+
alias="REQUEST_TIMEOUT_SECONDS",
|
| 191 |
+
)
|
| 192 |
+
rate_limit_requests_per_window: int = Field(
|
| 193 |
+
default=600,
|
| 194 |
+
ge=1,
|
| 195 |
+
alias="RATE_LIMIT_REQUESTS_PER_WINDOW",
|
| 196 |
+
)
|
| 197 |
+
internal_rate_limit_requests_per_window: int = Field(
|
| 198 |
+
default=120,
|
| 199 |
+
ge=1,
|
| 200 |
+
alias="INTERNAL_RATE_LIMIT_REQUESTS_PER_WINDOW",
|
| 201 |
+
)
|
| 202 |
+
rate_limit_window_seconds: int = Field(
|
| 203 |
+
default=60,
|
| 204 |
+
ge=1,
|
| 205 |
+
alias="RATE_LIMIT_WINDOW_SECONDS",
|
| 206 |
+
)
|
| 207 |
llm_timeout_seconds: int = Field(default=12, alias="LLM_TIMEOUT_SECONDS")
|
| 208 |
max_llm_concurrent_requests: int = Field(
|
| 209 |
default=4,
|
|
|
|
| 243 |
@model_validator(mode="after")
|
| 244 |
def validate_post_feed_security(self) -> "Settings":
|
| 245 |
self.vector_store_provider = self.vector_store_provider.strip().lower()
|
| 246 |
+
allowed_resources = {"places", "posts", "users", "clubs", "groups", "events"}
|
| 247 |
+
for resource_type, thresholds in self.global_search_resource_thresholds.items():
|
| 248 |
+
if resource_type not in allowed_resources:
|
| 249 |
+
raise ValueError(
|
| 250 |
+
f"GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON contiene un recurso invalido: {resource_type}"
|
| 251 |
+
)
|
| 252 |
+
if set(thresholds) != {"semantic_min", "lexical_min"}:
|
| 253 |
+
raise ValueError(
|
| 254 |
+
"cada override de GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON debe "
|
| 255 |
+
"contener semantic_min y lexical_min"
|
| 256 |
+
)
|
| 257 |
+
semantic_min = float(thresholds["semantic_min"])
|
| 258 |
+
lexical_min = float(thresholds["lexical_min"])
|
| 259 |
+
if not math.isfinite(semantic_min) or not -1.0 <= semantic_min <= 1.0:
|
| 260 |
+
raise ValueError("semantic_min debe ser finito y estar entre -1 y 1")
|
| 261 |
+
if not math.isfinite(lexical_min) or lexical_min < 0.0:
|
| 262 |
+
raise ValueError("lexical_min debe ser finito y no negativo")
|
| 263 |
+
if not math.isfinite(self.global_search_min_semantic_score):
|
| 264 |
+
raise ValueError("GLOBAL_SEARCH_MIN_SEMANTIC_SCORE debe ser finito")
|
| 265 |
+
if not math.isfinite(self.global_search_min_lexical_score):
|
| 266 |
+
raise ValueError("GLOBAL_SEARCH_MIN_LEXICAL_SCORE debe ser finito")
|
| 267 |
+
self.global_search_threshold_policy_version = (
|
| 268 |
+
self.global_search_threshold_policy_version.strip()
|
| 269 |
+
)
|
| 270 |
if self.env.lower() != "local":
|
| 271 |
missing: list[str] = []
|
| 272 |
if not self.post_feed_internal_token:
|
app/shared/search/source.py
CHANGED
|
@@ -34,6 +34,8 @@ class PagedMainApiSearchClient:
|
|
| 34 |
self._settings = settings
|
| 35 |
self._base_url = settings.main_api_base_url.rstrip("/") + "/"
|
| 36 |
self._path = path.lstrip("/")
|
|
|
|
|
|
|
| 37 |
self._collection_keys = collection_keys
|
| 38 |
self._mapper = mapper
|
| 39 |
self._page_limit = page_limit
|
|
@@ -88,8 +90,12 @@ class PagedMainApiSearchClient:
|
|
| 88 |
offset += limit
|
| 89 |
|
| 90 |
def _build_headers(self) -> dict[str, str]:
|
| 91 |
-
token = self._settings.main_api_internal_token or
|
| 92 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
|
| 94 |
def _pagination_params(
|
| 95 |
self,
|
|
|
|
| 34 |
self._settings = settings
|
| 35 |
self._base_url = settings.main_api_base_url.rstrip("/") + "/"
|
| 36 |
self._path = path.lstrip("/")
|
| 37 |
+
if not self._path.startswith("api/v1/internal/search/"):
|
| 38 |
+
raise ValueError("search sync source must use an internal snapshot endpoint")
|
| 39 |
self._collection_keys = collection_keys
|
| 40 |
self._mapper = mapper
|
| 41 |
self._page_limit = page_limit
|
|
|
|
| 90 |
offset += limit
|
| 91 |
|
| 92 |
def _build_headers(self) -> dict[str, str]:
|
| 93 |
+
token = (self._settings.main_api_internal_token or "").strip()
|
| 94 |
+
if not token:
|
| 95 |
+
raise RuntimeError(
|
| 96 |
+
"MAIN_API_INTERNAL_TOKEN is required for search snapshot synchronization"
|
| 97 |
+
)
|
| 98 |
+
return {"Authorization": f"Bearer {token}"}
|
| 99 |
|
| 100 |
def _pagination_params(
|
| 101 |
self,
|
app/shared/security/rate_limit.py
CHANGED
|
@@ -1,6 +1,65 @@
|
|
| 1 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
|
| 4 |
async def rate_limit_placeholder(request: Request) -> None:
|
| 5 |
-
"""
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import deque
|
| 2 |
+
from threading import Lock
|
| 3 |
+
from time import monotonic
|
| 4 |
+
|
| 5 |
+
from fastapi import HTTPException, Request, status
|
| 6 |
+
|
| 7 |
+
from app.shared.config.settings import get_settings
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
_requests: dict[str, deque[float]] = {}
|
| 11 |
+
_lock = Lock()
|
| 12 |
+
_MAX_BUCKETS = 10_000
|
| 13 |
|
| 14 |
|
| 15 |
async def rate_limit_placeholder(request: Request) -> None:
|
| 16 |
+
"""Apply a small in-process fixed-window guard at the HTTP boundary.
|
| 17 |
+
|
| 18 |
+
The production gateway may impose an additional distributed limit. This guard is
|
| 19 |
+
intentionally local to each worker so a missing gateway rule cannot leave the
|
| 20 |
+
service completely unbounded.
|
| 21 |
+
"""
|
| 22 |
+
settings = get_settings()
|
| 23 |
+
window_seconds = settings.rate_limit_window_seconds
|
| 24 |
+
limit = (
|
| 25 |
+
settings.internal_rate_limit_requests_per_window
|
| 26 |
+
if request.url.path.startswith("/internal/")
|
| 27 |
+
else settings.rate_limit_requests_per_window
|
| 28 |
+
)
|
| 29 |
+
client_host = request.client.host if request.client else "unknown"
|
| 30 |
+
key = f"{client_host}:{request.url.path}"
|
| 31 |
+
now = monotonic()
|
| 32 |
+
cutoff = now - window_seconds
|
| 33 |
+
|
| 34 |
+
with _lock:
|
| 35 |
+
if key not in _requests and len(_requests) >= _MAX_BUCKETS:
|
| 36 |
+
stale_keys = [
|
| 37 |
+
bucket_key
|
| 38 |
+
for bucket_key, bucket in _requests.items()
|
| 39 |
+
if not bucket or bucket[-1] <= cutoff
|
| 40 |
+
]
|
| 41 |
+
for stale_key in stale_keys:
|
| 42 |
+
_requests.pop(stale_key, None)
|
| 43 |
+
if len(_requests) >= _MAX_BUCKETS:
|
| 44 |
+
raise HTTPException(
|
| 45 |
+
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
| 46 |
+
detail="Rate limit capacity exceeded",
|
| 47 |
+
headers={"Retry-After": str(window_seconds)},
|
| 48 |
+
)
|
| 49 |
+
timestamps = _requests.setdefault(key, deque())
|
| 50 |
+
while timestamps and timestamps[0] <= cutoff:
|
| 51 |
+
timestamps.popleft()
|
| 52 |
+
if len(timestamps) >= limit:
|
| 53 |
+
retry_after = max(1, int(window_seconds - (now - timestamps[0])))
|
| 54 |
+
raise HTTPException(
|
| 55 |
+
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
| 56 |
+
detail="Rate limit exceeded",
|
| 57 |
+
headers={"Retry-After": str(retry_after)},
|
| 58 |
+
)
|
| 59 |
+
timestamps.append(now)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def reset_rate_limit_state() -> None:
|
| 63 |
+
"""Clear process-local buckets for deterministic tests."""
|
| 64 |
+
with _lock:
|
| 65 |
+
_requests.clear()
|
app/shared/vector_store/aws_pgvector.py
CHANGED
|
@@ -21,6 +21,13 @@ READ_CONTRACT_SIGNATURES = {
|
|
| 21 |
"get_post_feed_features": "get_post_feed_features(text, text[])",
|
| 22 |
}
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
SEARCH_RESOURCE_TYPES = frozenset({"places", "posts", "users", "clubs", "groups", "events"})
|
| 25 |
|
| 26 |
|
|
@@ -134,6 +141,20 @@ class AwsPgvectorClient:
|
|
| 134 |
"executable": bool(row["executable"]) if row else False,
|
| 135 |
}
|
| 136 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
except Exception as exc:
|
| 138 |
return {
|
| 139 |
"ready": False,
|
|
@@ -141,10 +162,7 @@ class AwsPgvectorClient:
|
|
| 141 |
"message": str(exc),
|
| 142 |
}
|
| 143 |
|
| 144 |
-
ready = vector_available
|
| 145 |
-
details["exists"] and details["executable"]
|
| 146 |
-
for details in functions.values()
|
| 147 |
-
)
|
| 148 |
return {
|
| 149 |
"ready": ready,
|
| 150 |
"vector_extension": vector_available,
|
|
@@ -387,9 +405,26 @@ def _build_connection_kwargs(settings: Settings, role: str) -> dict[str, Any]:
|
|
| 387 |
"database": settings.pgvector_database,
|
| 388 |
"user": user,
|
| 389 |
"password": password,
|
|
|
|
|
|
|
| 390 |
}
|
| 391 |
|
| 392 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 393 |
def _credentials_for_role(settings: Settings, role: str) -> tuple[str | None, str | None]:
|
| 394 |
if role == "writer":
|
| 395 |
return (
|
|
|
|
| 21 |
"get_post_feed_features": "get_post_feed_features(text, text[])",
|
| 22 |
}
|
| 23 |
|
| 24 |
+
SEARCH_CANDIDATE_V2_MARKERS = (
|
| 25 |
+
"event_active_at",
|
| 26 |
+
"min_semantic_score",
|
| 27 |
+
"min_lexical_score",
|
| 28 |
+
"try_parse_timestamptz",
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
SEARCH_RESOURCE_TYPES = frozenset({"places", "posts", "users", "clubs", "groups", "events"})
|
| 32 |
|
| 33 |
|
|
|
|
| 141 |
"executable": bool(row["executable"]) if row else False,
|
| 142 |
}
|
| 143 |
)
|
| 144 |
+
if function_name == "search_resource_embeddings":
|
| 145 |
+
definition_row = await connection.fetchrow(
|
| 146 |
+
"SELECT pg_get_functiondef(to_regprocedure($1)) AS body",
|
| 147 |
+
signature,
|
| 148 |
+
)
|
| 149 |
+
definition = (
|
| 150 |
+
str(definition_row["body"] or "")
|
| 151 |
+
if definition_row
|
| 152 |
+
else ""
|
| 153 |
+
)
|
| 154 |
+
functions[function_name]["candidate_filters_v2"] = all(
|
| 155 |
+
marker in definition
|
| 156 |
+
for marker in SEARCH_CANDIDATE_V2_MARKERS
|
| 157 |
+
)
|
| 158 |
except Exception as exc:
|
| 159 |
return {
|
| 160 |
"ready": False,
|
|
|
|
| 162 |
"message": str(exc),
|
| 163 |
}
|
| 164 |
|
| 165 |
+
ready = _read_contract_is_ready(vector_available, functions)
|
|
|
|
|
|
|
|
|
|
| 166 |
return {
|
| 167 |
"ready": ready,
|
| 168 |
"vector_extension": vector_available,
|
|
|
|
| 405 |
"database": settings.pgvector_database,
|
| 406 |
"user": user,
|
| 407 |
"password": password,
|
| 408 |
+
"timeout": settings.request_timeout_seconds,
|
| 409 |
+
"command_timeout": settings.request_timeout_seconds,
|
| 410 |
}
|
| 411 |
|
| 412 |
|
| 413 |
+
def _read_contract_is_ready(
|
| 414 |
+
vector_available: bool,
|
| 415 |
+
functions: dict[str, dict[str, Any]],
|
| 416 |
+
) -> bool:
|
| 417 |
+
return vector_available and all(
|
| 418 |
+
details.get("exists") is True
|
| 419 |
+
and details.get("executable") is True
|
| 420 |
+
and (
|
| 421 |
+
function_name != "search_resource_embeddings"
|
| 422 |
+
or details.get("candidate_filters_v2") is True
|
| 423 |
+
)
|
| 424 |
+
for function_name, details in functions.items()
|
| 425 |
+
)
|
| 426 |
+
|
| 427 |
+
|
| 428 |
def _credentials_for_role(settings: Settings, role: str) -> tuple[str | None, str | None]:
|
| 429 |
if role == "writer":
|
| 430 |
return (
|
docs/api_endpoints.md
CHANGED
|
@@ -42,6 +42,7 @@ Todas las peticiones y respuestas con body utilizan `application/json`.
|
|
| 42 |
| `GET` | `/internal/posts/clusters/runs/{id}` | Detalle de un run de clusters | Servicio |
|
| 43 |
| `POST` | `/internal/posts/clusters/runs/{id}/activate` | Activar/rollback de run | Servicio |
|
| 44 |
| `POST` | `/search` | Busqueda global sobre todos los recursos | Condicional |
|
|
|
|
| 45 |
|
| 46 |
La autenticacion de `/search` solo es obligatoria cuando el body contiene
|
| 47 |
`requester_id`. En ese caso se envia:
|
|
@@ -96,6 +97,7 @@ Comprueba que el servicio puede utilizar su almacenamiento vectorial. Cuando
|
|
| 96 |
- existencia y permiso de ejecucion de `match_places(...)`;
|
| 97 |
- existencia y permiso de ejecucion de `match_posts(...)`;
|
| 98 |
- existencia y permiso de ejecucion de `search_resource_embeddings(...)`.
|
|
|
|
| 99 |
|
| 100 |
Respuesta correcta: `200 OK` con `status: "ready"`.
|
| 101 |
|
|
@@ -759,19 +761,72 @@ Ejemplo de error de validacion `422`:
|
|
| 759 |
}
|
| 760 |
```
|
| 761 |
|
| 762 |
-
## 6.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 763 |
|
| 764 |
| Variable | Uso |
|
| 765 |
| --- | --- |
|
| 766 |
| `SEARCH_INTERNAL_TOKEN` | Valida `Authorization: Bearer ...` cuando `/search` recibe `requester_id` |
|
| 767 |
-
| `NLP_SERVICE_TOKEN` | Autentica
|
| 768 |
| `MAIN_API_INTERNAL_TOKEN` | Autentica los jobs NLP al leer endpoints internos de Go |
|
| 769 |
| `MAX_REQUEST_BODY_BYTES` | Debe ser al menos `131072`; valor recomendado `262144` para 500 candidatos |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 770 |
| `VECTOR_STORE_PROVIDER` | Selecciona `aws_pgvector` o el proveedor mock |
|
| 771 |
| `PGVECTOR_*` | Conexion y roles de PostgreSQL/pgvector |
|
| 772 |
| `EMBEDDING_PROVIDER` | Proveedor de embeddings; produccion utiliza `fasttext` |
|
| 773 |
| `EMBEDDING_DIMENSION` | Dimension vectorial; FastText utiliza `300` |
|
| 774 |
| `GROQ_API_KEY` | Habilita Groq/Llama para redactar respuestas conversacionales |
|
| 775 |
| `MAIN_API_BASE_URL` | API principal usada para fuentes y filtros geograficos |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 776 |
| `MAIN_API_PLACES_NEARBY_PATH` | Endpoint que resuelve los IDs dentro del radio solicitado |
|
| 777 |
| `GLOBAL_SEARCH_NEARBY_BOOST` | Peso de priorizacion geografica; default `0.12` |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
| `GET` | `/internal/posts/clusters/runs/{id}` | Detalle de un run de clusters | Servicio |
|
| 43 |
| `POST` | `/internal/posts/clusters/runs/{id}/activate` | Activar/rollback de run | Servicio |
|
| 44 |
| `POST` | `/search` | Busqueda global sobre todos los recursos | Condicional |
|
| 45 |
+
| `POST` | `/internal/search/candidates` | Candidatos tecnicos para la API principal | Servicio |
|
| 46 |
|
| 47 |
La autenticacion de `/search` solo es obligatoria cuando el body contiene
|
| 48 |
`requester_id`. En ese caso se envia:
|
|
|
|
| 97 |
- existencia y permiso de ejecucion de `match_places(...)`;
|
| 98 |
- existencia y permiso de ejecucion de `match_posts(...)`;
|
| 99 |
- existencia y permiso de ejecucion de `search_resource_embeddings(...)`.
|
| 100 |
+
- presencia de los filtros V2 (`event_active_at`, umbrales y parseo temporal seguro).
|
| 101 |
|
| 102 |
Respuesta correcta: `200 OK` con `status: "ready"`.
|
| 103 |
|
|
|
|
| 761 |
}
|
| 762 |
```
|
| 763 |
|
| 764 |
+
## 6. Candidatos internos de busqueda
|
| 765 |
+
|
| 766 |
+
### `POST /internal/search/candidates`
|
| 767 |
+
|
| 768 |
+
Endpoint de servicio consumido exclusivamente por la API principal. La app movil no
|
| 769 |
+
debe llamarlo directamente.
|
| 770 |
+
|
| 771 |
+
```http
|
| 772 |
+
Authorization: Bearer <NLP_SERVICE_TOKEN>
|
| 773 |
+
Content-Type: application/json
|
| 774 |
+
```
|
| 775 |
+
|
| 776 |
+
Ejemplo minimo:
|
| 777 |
+
|
| 778 |
+
```json
|
| 779 |
+
{
|
| 780 |
+
"query": "club universitario de ajedrez",
|
| 781 |
+
"resource_types": ["clubs", "groups", "events", "posts"],
|
| 782 |
+
"candidate_limit_per_type": 50,
|
| 783 |
+
"top_limit": 100,
|
| 784 |
+
"as_of": "2026-07-12T14:00:00Z"
|
| 785 |
+
}
|
| 786 |
+
```
|
| 787 |
+
|
| 788 |
+
La respuesta contiene secciones independientes con `id`, `resource_type`, `score`,
|
| 789 |
+
`semantic_score` y `lexical_score`. No incluye metadata de UI ni registros hidratados.
|
| 790 |
+
La API principal aplica permisos y reglas de negocio, carga los recursos vigentes y
|
| 791 |
+
conserva el orden del ranking.
|
| 792 |
+
|
| 793 |
+
NLP aplica antes del `LIMIT` los umbrales configurados. Un candidato se acepta cuando
|
| 794 |
+
supera el umbral semantico **o** el lexical; si ninguno lo supera se descarta y no se
|
| 795 |
+
rellena la respuesta. Para eventos tambien exige que
|
| 796 |
+
`start_time + duration_minutes > as_of`, evitando calcular ranking sobre eventos ya
|
| 797 |
+
finalizados. Los cursores quedan ligados a query, recursos, filtros, ubicacion,
|
| 798 |
+
`as_of` y version de la politica de umbrales.
|
| 799 |
+
|
| 800 |
+
## 7. Variables relacionadas
|
| 801 |
|
| 802 |
| Variable | Uso |
|
| 803 |
| --- | --- |
|
| 804 |
| `SEARCH_INTERNAL_TOKEN` | Valida `Authorization: Bearer ...` cuando `/search` recibe `requester_id` |
|
| 805 |
+
| `NLP_SERVICE_TOKEN` | Autentica `/internal/posts/*` y `/internal/search/candidates`; debe coincidir con `NLP_SERVICE_TOKEN` en Go |
|
| 806 |
| `MAIN_API_INTERNAL_TOKEN` | Autentica los jobs NLP al leer endpoints internos de Go |
|
| 807 |
| `MAX_REQUEST_BODY_BYTES` | Debe ser al menos `131072`; valor recomendado `262144` para 500 candidatos |
|
| 808 |
+
| `REQUEST_TIMEOUT_SECONDS` | Presupuesto maximo para search interno y timeout de conexion/consulta pgvector |
|
| 809 |
+
| `RATE_LIMIT_REQUESTS_PER_WINDOW` | Limite local por IP/ruta para endpoints publicos |
|
| 810 |
+
| `INTERNAL_RATE_LIMIT_REQUESTS_PER_WINDOW` | Limite local por IP/ruta para endpoints internos |
|
| 811 |
+
| `RATE_LIMIT_WINDOW_SECONDS` | Ventana del limitador local; el gateway puede agregar un limite distribuido |
|
| 812 |
| `VECTOR_STORE_PROVIDER` | Selecciona `aws_pgvector` o el proveedor mock |
|
| 813 |
| `PGVECTOR_*` | Conexion y roles de PostgreSQL/pgvector |
|
| 814 |
| `EMBEDDING_PROVIDER` | Proveedor de embeddings; produccion utiliza `fasttext` |
|
| 815 |
| `EMBEDDING_DIMENSION` | Dimension vectorial; FastText utiliza `300` |
|
| 816 |
| `GROQ_API_KEY` | Habilita Groq/Llama para redactar respuestas conversacionales |
|
| 817 |
| `MAIN_API_BASE_URL` | API principal usada para fuentes y filtros geograficos |
|
| 818 |
+
| `MAIN_API_USERS_SNAPSHOT_PATH` | Snapshot interno paginado de usuarios para search |
|
| 819 |
+
| `MAIN_API_CLUBS_SNAPSHOT_PATH` | Snapshot interno paginado de clubs para search |
|
| 820 |
+
| `MAIN_API_GROUPS_SNAPSHOT_PATH` | Snapshot interno paginado de grupos para search |
|
| 821 |
+
| `MAIN_API_EVENTS_SNAPSHOT_PATH` | Snapshot interno paginado de eventos para search |
|
| 822 |
| `MAIN_API_PLACES_NEARBY_PATH` | Endpoint que resuelve los IDs dentro del radio solicitado |
|
| 823 |
| `GLOBAL_SEARCH_NEARBY_BOOST` | Peso de priorizacion geografica; default `0.12` |
|
| 824 |
+
| `PUBLIC_GLOBAL_SEARCH_ENABLED` | Mantiene o deshabilita el endpoint publico heredado `POST /search` |
|
| 825 |
+
| `GLOBAL_SEARCH_MIN_SEMANTIC_SCORE` | Umbral semantico global; default `0.30` |
|
| 826 |
+
| `GLOBAL_SEARCH_MIN_LEXICAL_SCORE` | Umbral lexical global; default `0.05` |
|
| 827 |
+
| `GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON` | Overrides opcionales por recurso con `semantic_min` y `lexical_min` |
|
| 828 |
+
| `GLOBAL_SEARCH_THRESHOLD_POLICY_VERSION` | Version estable incluida en cursores para invalidarlos al cambiar la politica |
|
| 829 |
+
|
| 830 |
+
`top_limit` se conserva en el request y en la huella del cursor para mantener el
|
| 831 |
+
contexto coordinado con Go. NLP devuelve candidatos por seccion; la composicion final de
|
| 832 |
+
`top_results` pertenece a la API principal.
|
docs/global_search.md
CHANGED
|
@@ -1,54 +1,142 @@
|
|
| 1 |
# Busqueda global
|
| 2 |
|
| 3 |
-
##
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
-
|
| 8 |
-
|
| 9 |
-
-
|
| 10 |
-
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
-
|
| 17 |
-
|
| 18 |
-
- `
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# Busqueda global
|
| 2 |
|
| 3 |
+
## Fuente de verdad y flujo
|
| 4 |
+
|
| 5 |
+
```text
|
| 6 |
+
App movil + JWT
|
| 7 |
+
-> API principal Go: POST /api/v1/search
|
| 8 |
+
-> NLP: POST /internal/search/candidates + NLP_SERVICE_TOKEN
|
| 9 |
+
-> pgvector/FTS: IDs y scores sobre umbral
|
| 10 |
+
-> Go: permisos, reglas de negocio e hidratacion PostgreSQL
|
| 11 |
+
-> App movil: recursos completos
|
| 12 |
+
```
|
| 13 |
+
|
| 14 |
+
- La API principal conserva identidad, permisos y objetos originales.
|
| 15 |
+
- NLP solo calcula embeddings, recuperacion, umbrales, ranking y cursores internos.
|
| 16 |
+
- PostgreSQL pgvector contiene indices derivados reconstruibles.
|
| 17 |
+
- La respuesta interna excluye metadata privada y campos para presentacion.
|
| 18 |
+
- `POST /search` es compatibilidad temporal y puede apagarse con
|
| 19 |
+
`PUBLIC_GLOBAL_SEARCH_ENABLED=false`.
|
| 20 |
+
|
| 21 |
+
## Relevancia sin relleno
|
| 22 |
+
|
| 23 |
+
Cada recurso posee `semantic_min` y `lexical_min`. Un candidato se acepta cuando cumple
|
| 24 |
+
al menos una de estas condiciones:
|
| 25 |
+
|
| 26 |
+
```text
|
| 27 |
+
semantic_score >= semantic_min
|
| 28 |
+
OR lexical_score >= lexical_min
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
RRF ordena candidatos aceptados, pero no decide por si solo si existe coincidencia. El
|
| 32 |
+
limite solicitado es un maximo: cero coincidencias devuelve `[]`; una coincidencia
|
| 33 |
+
devuelve una. La diversificacion nunca recupera elementos bajo el umbral.
|
| 34 |
+
|
| 35 |
+
Variables:
|
| 36 |
+
|
| 37 |
+
```dotenv
|
| 38 |
+
GLOBAL_SEARCH_MIN_SEMANTIC_SCORE=0.30
|
| 39 |
+
GLOBAL_SEARCH_MIN_LEXICAL_SCORE=0.05
|
| 40 |
+
GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON={}
|
| 41 |
+
GLOBAL_SEARCH_THRESHOLD_POLICY_VERSION=global-search-relevance-v1
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
Ejemplo de override completo:
|
| 45 |
+
|
| 46 |
+
```dotenv
|
| 47 |
+
GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON={"posts":{"semantic_min":0.35,"lexical_min":0.05},"events":{"semantic_min":0.38,"lexical_min":0.06}}
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
Los valores iniciales deben recalibrarse con consultas positivas y negativas reales. El
|
| 51 |
+
umbral lexical corresponde a `ts_rank_cd`; no se debe copiar
|
| 52 |
+
`BM25_RELEVANCE_THRESHOLD`.
|
| 53 |
+
|
| 54 |
+
El repositorio incluye un calibrador reproducible que solo consume etiquetas y scores,
|
| 55 |
+
sin texto ni metadata privada. Cada linea JSONL debe usar este contrato:
|
| 56 |
+
|
| 57 |
+
```json
|
| 58 |
+
{"resource_type":"events","relevant":true,"semantic_score":0.62,"lexical_score":0.08}
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Ejecucion:
|
| 62 |
+
|
| 63 |
+
```powershell
|
| 64 |
+
python scripts/calibrate_global_search_thresholds.py .\global_search_labels.jsonl `
|
| 65 |
+
--policy-version global-search-relevance-v2 `
|
| 66 |
+
--target-precision 0.95 `
|
| 67 |
+
--output .\global_search_calibration.json
|
| 68 |
+
```
|
| 69 |
+
|
| 70 |
+
El resultado contiene `resource_thresholds` listo para copiar a
|
| 71 |
+
`GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON` y metricas por recurso. La activacion sigue
|
| 72 |
+
bloqueada hasta alimentar el script con resultados reales de los seis recursos.
|
| 73 |
+
|
| 74 |
+
## Eventos
|
| 75 |
+
|
| 76 |
+
El endpoint interno recibe `as_of` desde Go y pgvector aplica en el CTE `eligible`, antes
|
| 77 |
+
del ranking:
|
| 78 |
+
|
| 79 |
+
```text
|
| 80 |
+
start_time + duration_minutes > as_of
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
Esto conserva eventos futuros y en curso, y elimina eventos terminados o con metadata
|
| 84 |
+
temporal invalida. Go repite la validacion con la tabla principal; NLP solo optimiza.
|
| 85 |
+
|
| 86 |
+
## Ubicacion, filtros y cursores
|
| 87 |
+
|
| 88 |
+
- `prioritize` agrega una senal separada de cercania.
|
| 89 |
+
- `strict` limita places, clubs y events a los `place_ids` resueltos por la API principal.
|
| 90 |
+
- Los filtros duros se ejecutan en SQL y se validan nuevamente en Python.
|
| 91 |
+
- Cada recurso tiene cursor independiente ligado a query, filtros, ubicacion, `as_of`,
|
| 92 |
+
limite y version de la politica.
|
| 93 |
+
- Go encapsula el cursor NLP dentro de su cursor HMAC; la app nunca recibe el cursor
|
| 94 |
+
interno sin esa proteccion.
|
| 95 |
+
|
| 96 |
+
## Migracion de una BD existente
|
| 97 |
+
|
| 98 |
+
Ejecutar con el propietario de `search_resource_embeddings(...)`:
|
| 99 |
+
|
| 100 |
+
1. `sql/migrations/20260712_01_global_search_candidate_filters.sql`.
|
| 101 |
+
2. `sql/verify_global_search_candidate_filters.sql`.
|
| 102 |
+
|
| 103 |
+
El primer archivo es el unico incremental de esta funcionalidad. No crea tablas,
|
| 104 |
+
columnas ni indices; usa `CREATE OR REPLACE FUNCTION`, conserva la firma, agrega helpers
|
| 105 |
+
de parseo seguro y vuelve a otorgar `EXECUTE` a `nlp_reader`. Sus postchecks funcionales
|
| 106 |
+
ocurren antes del `COMMIT`. El verificador posterior abre una transaccion de solo lectura
|
| 107 |
+
y aborta si encuentra contrato viejo, consultas bajo umbral con resultados, eventos
|
| 108 |
+
terminados o eventos activos con metadata temporal invalida.
|
| 109 |
+
|
| 110 |
+
No ejecutar `sql/new_pgvector_schema.sql` en produccion: es el esquema convergente para
|
| 111 |
+
inicializar desde cero. `sql/aws_pgvector_contract.sql` contiene el contrato completo de
|
| 112 |
+
referencia.
|
| 113 |
+
|
| 114 |
+
## Sincronizacion
|
| 115 |
+
|
| 116 |
+
Las fuentes para users, clubs, groups y events son endpoints internos paginados de Go,
|
| 117 |
+
autenticados con `MAIN_API_INTERNAL_TOKEN`:
|
| 118 |
+
|
| 119 |
+
```dotenv
|
| 120 |
+
MAIN_API_USERS_SNAPSHOT_PATH=/api/v1/internal/search/users/snapshot
|
| 121 |
+
MAIN_API_CLUBS_SNAPSHOT_PATH=/api/v1/internal/search/clubs/snapshot
|
| 122 |
+
MAIN_API_GROUPS_SNAPSHOT_PATH=/api/v1/internal/search/groups/snapshot
|
| 123 |
+
MAIN_API_EVENTS_SNAPSHOT_PATH=/api/v1/internal/search/events/snapshot
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
Los jobs de search no aceptan `MAIN_API_AUTH_TOKEN` como reemplazo. La identidad de un
|
| 127 |
+
usuario final no debe controlar la construccion del indice derivado.
|
| 128 |
+
|
| 129 |
+
Antes de activar la busqueda:
|
| 130 |
+
|
| 131 |
+
1. Ejecutar dry-run de `python -m app.jobs.sync_search_embeddings --resource all`.
|
| 132 |
+
2. Sincronizar todos los recursos.
|
| 133 |
+
3. Confirmar que events incluye `start_time` y `duration_minutes` validos.
|
| 134 |
+
4. Ejecutar `sql/verify_global_search_embeddings.sql`.
|
| 135 |
+
5. Ejecutar el verificador V2 indicado arriba.
|
| 136 |
+
6. Desplegar NLP y probar `/health`, `/ready` e `/internal/search/candidates`.
|
| 137 |
+
7. Activar `GLOBAL_SEARCH_ENABLED=true` en Go.
|
| 138 |
+
|
| 139 |
+
`/ready` devuelve `503` si la firma existe pero la definicion instalada no contiene el
|
| 140 |
+
contrato V2. Search interno tambien posee timeout total y las consultas pgvector usan el
|
| 141 |
+
mismo presupuesto. El rate limit local es una ultima defensa por proceso; en despliegues
|
| 142 |
+
con multiples replicas debe complementarse con el gateway.
|
scripts/calibrate_global_search_thresholds.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Calibrate per-resource global-search thresholds from labeled score samples.
|
| 2 |
+
|
| 3 |
+
Input is JSONL with: resource_type, relevant, semantic_score and lexical_score.
|
| 4 |
+
At least one score must be present in every row. The script never reads source text or
|
| 5 |
+
private metadata.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import math
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from typing import Iterable
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
RESOURCE_TYPES = ("places", "posts", "users", "clubs", "groups", "events")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True)
|
| 22 |
+
class LabeledScore:
|
| 23 |
+
resource_type: str
|
| 24 |
+
relevant: bool
|
| 25 |
+
semantic_score: float | None
|
| 26 |
+
lexical_score: float | None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def calibrate_resource(
|
| 30 |
+
samples: list[LabeledScore],
|
| 31 |
+
target_precision: float = 0.95,
|
| 32 |
+
default_semantic: float = 0.30,
|
| 33 |
+
default_lexical: float = 0.05,
|
| 34 |
+
) -> dict[str, float | int]:
|
| 35 |
+
positives = sum(sample.relevant for sample in samples)
|
| 36 |
+
negatives = len(samples) - positives
|
| 37 |
+
if positives == 0 or negatives == 0:
|
| 38 |
+
raise ValueError("each resource requires positive and negative labeled samples")
|
| 39 |
+
|
| 40 |
+
semantic_candidates = _candidate_values(
|
| 41 |
+
(sample.semantic_score for sample in samples),
|
| 42 |
+
default=default_semantic,
|
| 43 |
+
minimum=-1.0,
|
| 44 |
+
maximum=1.0,
|
| 45 |
+
)
|
| 46 |
+
lexical_candidates = _candidate_values(
|
| 47 |
+
(sample.lexical_score for sample in samples),
|
| 48 |
+
default=default_lexical,
|
| 49 |
+
minimum=0.0,
|
| 50 |
+
maximum=None,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
evaluated: list[dict[str, float | int]] = []
|
| 54 |
+
for semantic_min in semantic_candidates:
|
| 55 |
+
for lexical_min in lexical_candidates:
|
| 56 |
+
tp = fp = fn = 0
|
| 57 |
+
for sample in samples:
|
| 58 |
+
accepted = (
|
| 59 |
+
sample.semantic_score is not None
|
| 60 |
+
and sample.semantic_score >= semantic_min
|
| 61 |
+
) or (
|
| 62 |
+
sample.lexical_score is not None
|
| 63 |
+
and sample.lexical_score >= lexical_min
|
| 64 |
+
)
|
| 65 |
+
if accepted and sample.relevant:
|
| 66 |
+
tp += 1
|
| 67 |
+
elif accepted:
|
| 68 |
+
fp += 1
|
| 69 |
+
elif sample.relevant:
|
| 70 |
+
fn += 1
|
| 71 |
+
precision = tp / (tp + fp) if tp + fp else 1.0
|
| 72 |
+
recall = tp / (tp + fn) if tp + fn else 0.0
|
| 73 |
+
f1 = (
|
| 74 |
+
2.0 * precision * recall / (precision + recall)
|
| 75 |
+
if precision + recall
|
| 76 |
+
else 0.0
|
| 77 |
+
)
|
| 78 |
+
evaluated.append(
|
| 79 |
+
{
|
| 80 |
+
"semantic_min": semantic_min,
|
| 81 |
+
"lexical_min": lexical_min,
|
| 82 |
+
"precision": precision,
|
| 83 |
+
"recall": recall,
|
| 84 |
+
"f1": f1,
|
| 85 |
+
"true_positives": tp,
|
| 86 |
+
"false_positives": fp,
|
| 87 |
+
"false_negatives": fn,
|
| 88 |
+
}
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
precision_qualified = [
|
| 92 |
+
result for result in evaluated if result["precision"] >= target_precision
|
| 93 |
+
]
|
| 94 |
+
pool = precision_qualified or evaluated
|
| 95 |
+
return max(
|
| 96 |
+
pool,
|
| 97 |
+
key=lambda result: (
|
| 98 |
+
result["recall"] if precision_qualified else result["f1"],
|
| 99 |
+
result["precision"],
|
| 100 |
+
result["f1"],
|
| 101 |
+
-result["semantic_min"],
|
| 102 |
+
-result["lexical_min"],
|
| 103 |
+
),
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def load_samples(path: Path) -> list[LabeledScore]:
|
| 108 |
+
samples: list[LabeledScore] = []
|
| 109 |
+
for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
| 110 |
+
if not raw_line.strip():
|
| 111 |
+
continue
|
| 112 |
+
try:
|
| 113 |
+
payload = json.loads(raw_line)
|
| 114 |
+
resource_type = str(payload["resource_type"])
|
| 115 |
+
relevant = payload["relevant"]
|
| 116 |
+
if resource_type not in RESOURCE_TYPES or not isinstance(relevant, bool):
|
| 117 |
+
raise ValueError("invalid resource_type or relevant label")
|
| 118 |
+
semantic = _optional_finite_float(payload.get("semantic_score"))
|
| 119 |
+
lexical = _optional_finite_float(payload.get("lexical_score"))
|
| 120 |
+
if semantic is None and lexical is None:
|
| 121 |
+
raise ValueError("at least one score is required")
|
| 122 |
+
if semantic is not None and not -1.0 <= semantic <= 1.0:
|
| 123 |
+
raise ValueError("semantic_score must be between -1 and 1")
|
| 124 |
+
if lexical is not None and lexical < 0.0:
|
| 125 |
+
raise ValueError("lexical_score must be non-negative")
|
| 126 |
+
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
| 127 |
+
raise ValueError(f"invalid calibration row {line_number}: {exc}") from exc
|
| 128 |
+
samples.append(LabeledScore(resource_type, relevant, semantic, lexical))
|
| 129 |
+
return samples
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _optional_finite_float(value: object) -> float | None:
|
| 133 |
+
if value is None:
|
| 134 |
+
return None
|
| 135 |
+
parsed = float(value)
|
| 136 |
+
if not math.isfinite(parsed):
|
| 137 |
+
raise ValueError("scores must be finite")
|
| 138 |
+
return parsed
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _candidate_values(
|
| 142 |
+
values: Iterable[float | None],
|
| 143 |
+
*,
|
| 144 |
+
default: float,
|
| 145 |
+
minimum: float,
|
| 146 |
+
maximum: float | None,
|
| 147 |
+
) -> list[float]:
|
| 148 |
+
observed = sorted({value for value in values if value is not None})
|
| 149 |
+
if len(observed) > 200:
|
| 150 |
+
observed = [
|
| 151 |
+
observed[round(index * (len(observed) - 1) / 199)]
|
| 152 |
+
for index in range(200)
|
| 153 |
+
]
|
| 154 |
+
candidates = {default, minimum, *observed}
|
| 155 |
+
if observed:
|
| 156 |
+
above_max = math.nextafter(observed[-1], math.inf)
|
| 157 |
+
if maximum is None or above_max <= maximum:
|
| 158 |
+
candidates.add(above_max)
|
| 159 |
+
if maximum is not None:
|
| 160 |
+
candidates.add(maximum)
|
| 161 |
+
return sorted(
|
| 162 |
+
value
|
| 163 |
+
for value in candidates
|
| 164 |
+
if value >= minimum and (maximum is None or value <= maximum)
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def main() -> None:
|
| 169 |
+
parser = argparse.ArgumentParser()
|
| 170 |
+
parser.add_argument("input", type=Path)
|
| 171 |
+
parser.add_argument("--policy-version", required=True)
|
| 172 |
+
parser.add_argument("--target-precision", type=float, default=0.95)
|
| 173 |
+
parser.add_argument("--allow-partial", action="store_true")
|
| 174 |
+
parser.add_argument("--output", type=Path)
|
| 175 |
+
args = parser.parse_args()
|
| 176 |
+
if not 0.0 < args.target_precision <= 1.0:
|
| 177 |
+
raise SystemExit("--target-precision must be in (0, 1]")
|
| 178 |
+
|
| 179 |
+
samples = load_samples(args.input)
|
| 180 |
+
grouped = {
|
| 181 |
+
resource_type: [
|
| 182 |
+
sample for sample in samples if sample.resource_type == resource_type
|
| 183 |
+
]
|
| 184 |
+
for resource_type in RESOURCE_TYPES
|
| 185 |
+
}
|
| 186 |
+
missing = [resource for resource, items in grouped.items() if not items]
|
| 187 |
+
if missing and not args.allow_partial:
|
| 188 |
+
raise SystemExit(f"missing labeled resources: {', '.join(missing)}")
|
| 189 |
+
|
| 190 |
+
results = {
|
| 191 |
+
resource: calibrate_resource(items, target_precision=args.target_precision)
|
| 192 |
+
for resource, items in grouped.items()
|
| 193 |
+
if items
|
| 194 |
+
}
|
| 195 |
+
output = {
|
| 196 |
+
"policy_version": args.policy_version,
|
| 197 |
+
"target_precision": args.target_precision,
|
| 198 |
+
"resource_thresholds": {
|
| 199 |
+
resource: {
|
| 200 |
+
"semantic_min": result["semantic_min"],
|
| 201 |
+
"lexical_min": result["lexical_min"],
|
| 202 |
+
}
|
| 203 |
+
for resource, result in results.items()
|
| 204 |
+
},
|
| 205 |
+
"metrics": results,
|
| 206 |
+
}
|
| 207 |
+
rendered = json.dumps(output, ensure_ascii=False, indent=2, sort_keys=True)
|
| 208 |
+
if args.output:
|
| 209 |
+
args.output.write_text(rendered + "\n", encoding="utf-8")
|
| 210 |
+
else:
|
| 211 |
+
print(rendered)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
if __name__ == "__main__":
|
| 215 |
+
main()
|
sql/aws_pgvector_contract.sql
CHANGED
|
@@ -393,6 +393,37 @@ AS $$
|
|
| 393 |
WHERE p.external_id = ANY(p_external_ids);
|
| 394 |
$$;
|
| 395 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 396 |
CREATE OR REPLACE FUNCTION search_resource_embeddings(
|
| 397 |
p_resource_type TEXT,
|
| 398 |
p_query_text TEXT,
|
|
@@ -462,24 +493,40 @@ BEGIN
|
|
| 462 |
)
|
| 463 |
AND (
|
| 464 |
($4 ? 'published_from') IS FALSE
|
| 465 |
-
OR
|
| 466 |
>= ($4->>'published_from')::timestamptz
|
| 467 |
)
|
| 468 |
AND (
|
| 469 |
($4 ? 'published_to') IS FALSE
|
| 470 |
-
OR
|
| 471 |
<= ($4->>'published_to')::timestamptz
|
| 472 |
)
|
| 473 |
AND (
|
| 474 |
($4 ? 'event_from') IS FALSE
|
| 475 |
-
OR
|
| 476 |
>= ($4->>'event_from')::timestamptz
|
| 477 |
)
|
| 478 |
AND (
|
| 479 |
($4 ? 'event_to') IS FALSE
|
| 480 |
-
OR
|
| 481 |
<= ($4->>'event_to')::timestamptz
|
| 482 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 483 |
AND (
|
| 484 |
($4 ? 'is_online') IS FALSE
|
| 485 |
OR COALESCE((e.metadata->>'is_online')::boolean, false)
|
|
@@ -570,6 +617,26 @@ BEGIN
|
|
| 570 |
f.lexical_score::double precision
|
| 571 |
FROM fused f
|
| 572 |
JOIN eligible e USING (external_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 573 |
ORDER BY score DESC, f.semantic_score DESC NULLS LAST, e.external_id ASC
|
| 574 |
LIMIT GREATEST($3, 1)
|
| 575 |
$query$, target_table)
|
|
@@ -577,6 +644,11 @@ BEGIN
|
|
| 577 |
END;
|
| 578 |
$$;
|
| 579 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 580 |
CREATE OR REPLACE FUNCTION get_resource_content_hashes(
|
| 581 |
p_resource_type TEXT,
|
| 582 |
p_external_ids TEXT[]
|
|
|
|
| 393 |
WHERE p.external_id = ANY(p_external_ids);
|
| 394 |
$$;
|
| 395 |
|
| 396 |
+
CREATE OR REPLACE FUNCTION try_parse_timestamptz(p_value TEXT)
|
| 397 |
+
RETURNS TIMESTAMPTZ
|
| 398 |
+
LANGUAGE plpgsql
|
| 399 |
+
IMMUTABLE
|
| 400 |
+
PARALLEL SAFE
|
| 401 |
+
SET search_path = pg_catalog
|
| 402 |
+
AS $$
|
| 403 |
+
BEGIN
|
| 404 |
+
RETURN NULLIF(btrim(p_value), '')::timestamptz;
|
| 405 |
+
EXCEPTION WHEN OTHERS THEN
|
| 406 |
+
RETURN NULL;
|
| 407 |
+
END;
|
| 408 |
+
$$;
|
| 409 |
+
|
| 410 |
+
CREATE OR REPLACE FUNCTION try_parse_positive_integer(p_value TEXT)
|
| 411 |
+
RETURNS INTEGER
|
| 412 |
+
LANGUAGE plpgsql
|
| 413 |
+
IMMUTABLE
|
| 414 |
+
PARALLEL SAFE
|
| 415 |
+
SET search_path = pg_catalog
|
| 416 |
+
AS $$
|
| 417 |
+
DECLARE
|
| 418 |
+
parsed INTEGER;
|
| 419 |
+
BEGIN
|
| 420 |
+
parsed := NULLIF(btrim(p_value), '')::integer;
|
| 421 |
+
RETURN CASE WHEN parsed > 0 THEN parsed ELSE NULL END;
|
| 422 |
+
EXCEPTION WHEN OTHERS THEN
|
| 423 |
+
RETURN NULL;
|
| 424 |
+
END;
|
| 425 |
+
$$;
|
| 426 |
+
|
| 427 |
CREATE OR REPLACE FUNCTION search_resource_embeddings(
|
| 428 |
p_resource_type TEXT,
|
| 429 |
p_query_text TEXT,
|
|
|
|
| 493 |
)
|
| 494 |
AND (
|
| 495 |
($4 ? 'published_from') IS FALSE
|
| 496 |
+
OR public.try_parse_timestamptz(e.metadata->>'published_at')
|
| 497 |
>= ($4->>'published_from')::timestamptz
|
| 498 |
)
|
| 499 |
AND (
|
| 500 |
($4 ? 'published_to') IS FALSE
|
| 501 |
+
OR public.try_parse_timestamptz(e.metadata->>'published_at')
|
| 502 |
<= ($4->>'published_to')::timestamptz
|
| 503 |
)
|
| 504 |
AND (
|
| 505 |
($4 ? 'event_from') IS FALSE
|
| 506 |
+
OR public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 507 |
>= ($4->>'event_from')::timestamptz
|
| 508 |
)
|
| 509 |
AND (
|
| 510 |
($4 ? 'event_to') IS FALSE
|
| 511 |
+
OR public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 512 |
<= ($4->>'event_to')::timestamptz
|
| 513 |
)
|
| 514 |
+
AND (
|
| 515 |
+
($4 ? 'event_active_at') IS FALSE
|
| 516 |
+
OR (
|
| 517 |
+
public.try_parse_timestamptz(e.metadata->>'start_time') IS NOT NULL
|
| 518 |
+
AND public.try_parse_positive_integer(
|
| 519 |
+
e.metadata->>'duration_minutes'
|
| 520 |
+
) IS NOT NULL
|
| 521 |
+
AND public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 522 |
+
+ make_interval(
|
| 523 |
+
mins => public.try_parse_positive_integer(
|
| 524 |
+
e.metadata->>'duration_minutes'
|
| 525 |
+
)
|
| 526 |
+
)
|
| 527 |
+
> ($4->>'event_active_at')::timestamptz
|
| 528 |
+
)
|
| 529 |
+
)
|
| 530 |
AND (
|
| 531 |
($4 ? 'is_online') IS FALSE
|
| 532 |
OR COALESCE((e.metadata->>'is_online')::boolean, false)
|
|
|
|
| 617 |
f.lexical_score::double precision
|
| 618 |
FROM fused f
|
| 619 |
JOIN eligible e USING (external_id)
|
| 620 |
+
WHERE (
|
| 621 |
+
(
|
| 622 |
+
($4 ? 'min_semantic_score') IS FALSE
|
| 623 |
+
AND ($4 ? 'min_lexical_score') IS FALSE
|
| 624 |
+
)
|
| 625 |
+
OR (
|
| 626 |
+
($4 ? 'min_semantic_score')
|
| 627 |
+
AND COALESCE(
|
| 628 |
+
f.semantic_score >= ($4->>'min_semantic_score')::double precision,
|
| 629 |
+
FALSE
|
| 630 |
+
)
|
| 631 |
+
)
|
| 632 |
+
OR (
|
| 633 |
+
($4 ? 'min_lexical_score')
|
| 634 |
+
AND COALESCE(
|
| 635 |
+
f.lexical_score >= ($4->>'min_lexical_score')::double precision,
|
| 636 |
+
FALSE
|
| 637 |
+
)
|
| 638 |
+
)
|
| 639 |
+
)
|
| 640 |
ORDER BY score DESC, f.semantic_score DESC NULLS LAST, e.external_id ASC
|
| 641 |
LIMIT GREATEST($3, 1)
|
| 642 |
$query$, target_table)
|
|
|
|
| 644 |
END;
|
| 645 |
$$;
|
| 646 |
|
| 647 |
+
REVOKE ALL ON FUNCTION public.try_parse_timestamptz(text) FROM PUBLIC;
|
| 648 |
+
REVOKE ALL ON FUNCTION public.try_parse_positive_integer(text) FROM PUBLIC;
|
| 649 |
+
GRANT EXECUTE ON FUNCTION public.try_parse_timestamptz(text) TO nlp_reader;
|
| 650 |
+
GRANT EXECUTE ON FUNCTION public.try_parse_positive_integer(text) TO nlp_reader;
|
| 651 |
+
|
| 652 |
CREATE OR REPLACE FUNCTION get_resource_content_hashes(
|
| 653 |
p_resource_type TEXT,
|
| 654 |
p_external_ids TEXT[]
|
sql/migrations/20260712_01_global_search_candidate_filters.sql
ADDED
|
@@ -0,0 +1,413 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- Busqueda global V2: vigencia de eventos y umbrales de relevancia.
|
| 2 |
+
--
|
| 3 |
+
-- Ejecutar UNA sola vez en la BD NLP desplegada con el propietario de la
|
| 4 |
+
-- funcion public.search_resource_embeddings. No crea tablas, columnas ni
|
| 5 |
+
-- indices; conserva la firma usada por la API y agrega dos helpers de parseo
|
| 6 |
+
-- seguro para que metadata temporal corrupta se descarte sin abortar la busqueda.
|
| 7 |
+
|
| 8 |
+
BEGIN;
|
| 9 |
+
|
| 10 |
+
SET LOCAL lock_timeout = '10s';
|
| 11 |
+
SET LOCAL statement_timeout = '10min';
|
| 12 |
+
SET LOCAL idle_in_transaction_session_timeout = '5min';
|
| 13 |
+
SET LOCAL search_path = public;
|
| 14 |
+
|
| 15 |
+
SELECT pg_advisory_xact_lock(
|
| 16 |
+
hashtextextended('public:20260712_01_global_search_candidate_filters', 0)
|
| 17 |
+
);
|
| 18 |
+
|
| 19 |
+
DO $preflight$
|
| 20 |
+
DECLARE
|
| 21 |
+
required_table TEXT;
|
| 22 |
+
current_definition TEXT;
|
| 23 |
+
BEGIN
|
| 24 |
+
IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') THEN
|
| 25 |
+
RAISE EXCEPTION 'Preflight search V2: falta la extension vector';
|
| 26 |
+
END IF;
|
| 27 |
+
|
| 28 |
+
IF to_regprocedure(
|
| 29 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)'
|
| 30 |
+
) IS NULL THEN
|
| 31 |
+
RAISE EXCEPTION 'Preflight search V2: falta search_resource_embeddings(text,text,vector,integer,jsonb)';
|
| 32 |
+
END IF;
|
| 33 |
+
|
| 34 |
+
FOREACH required_table IN ARRAY ARRAY[
|
| 35 |
+
'place_embeddings',
|
| 36 |
+
'post_embeddings',
|
| 37 |
+
'user_embeddings',
|
| 38 |
+
'club_embeddings',
|
| 39 |
+
'group_embeddings',
|
| 40 |
+
'event_embeddings'
|
| 41 |
+
] LOOP
|
| 42 |
+
IF to_regclass('public.' || required_table) IS NULL THEN
|
| 43 |
+
RAISE EXCEPTION 'Preflight search V2: falta public.%', required_table;
|
| 44 |
+
END IF;
|
| 45 |
+
END LOOP;
|
| 46 |
+
|
| 47 |
+
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'nlp_reader') THEN
|
| 48 |
+
RAISE EXCEPTION 'Preflight search V2: falta el rol nlp_reader';
|
| 49 |
+
END IF;
|
| 50 |
+
|
| 51 |
+
SELECT pg_get_functiondef(
|
| 52 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)'::regprocedure
|
| 53 |
+
) INTO current_definition;
|
| 54 |
+
|
| 55 |
+
IF current_definition LIKE '%event_active_at%'
|
| 56 |
+
OR current_definition LIKE '%min_semantic_score%'
|
| 57 |
+
OR current_definition LIKE '%min_lexical_score%' THEN
|
| 58 |
+
RAISE EXCEPTION USING
|
| 59 |
+
MESSAGE = 'Preflight search V2: la funcion ya contiene cambios V2 o una instalacion parcial',
|
| 60 |
+
HINT = 'No vuelva a ejecutar la migracion. Ejecute sql/verify_global_search_candidate_filters.sql.';
|
| 61 |
+
END IF;
|
| 62 |
+
END;
|
| 63 |
+
$preflight$;
|
| 64 |
+
|
| 65 |
+
CREATE OR REPLACE FUNCTION public.try_parse_timestamptz(p_value TEXT)
|
| 66 |
+
RETURNS TIMESTAMPTZ
|
| 67 |
+
LANGUAGE plpgsql
|
| 68 |
+
IMMUTABLE
|
| 69 |
+
PARALLEL SAFE
|
| 70 |
+
SET search_path = pg_catalog
|
| 71 |
+
AS $$
|
| 72 |
+
BEGIN
|
| 73 |
+
RETURN NULLIF(btrim(p_value), '')::timestamptz;
|
| 74 |
+
EXCEPTION WHEN OTHERS THEN
|
| 75 |
+
RETURN NULL;
|
| 76 |
+
END;
|
| 77 |
+
$$;
|
| 78 |
+
|
| 79 |
+
CREATE OR REPLACE FUNCTION public.try_parse_positive_integer(p_value TEXT)
|
| 80 |
+
RETURNS INTEGER
|
| 81 |
+
LANGUAGE plpgsql
|
| 82 |
+
IMMUTABLE
|
| 83 |
+
PARALLEL SAFE
|
| 84 |
+
SET search_path = pg_catalog
|
| 85 |
+
AS $$
|
| 86 |
+
DECLARE
|
| 87 |
+
parsed INTEGER;
|
| 88 |
+
BEGIN
|
| 89 |
+
parsed := NULLIF(btrim(p_value), '')::integer;
|
| 90 |
+
RETURN CASE WHEN parsed > 0 THEN parsed ELSE NULL END;
|
| 91 |
+
EXCEPTION WHEN OTHERS THEN
|
| 92 |
+
RETURN NULL;
|
| 93 |
+
END;
|
| 94 |
+
$$;
|
| 95 |
+
|
| 96 |
+
CREATE OR REPLACE FUNCTION public.search_resource_embeddings(
|
| 97 |
+
p_resource_type TEXT,
|
| 98 |
+
p_query_text TEXT,
|
| 99 |
+
p_query_embedding VECTOR(300),
|
| 100 |
+
p_match_count INTEGER,
|
| 101 |
+
p_filters JSONB DEFAULT '{}'::jsonb
|
| 102 |
+
)
|
| 103 |
+
RETURNS TABLE (
|
| 104 |
+
external_id TEXT,
|
| 105 |
+
document TEXT,
|
| 106 |
+
metadata JSONB,
|
| 107 |
+
score DOUBLE PRECISION,
|
| 108 |
+
semantic_score DOUBLE PRECISION,
|
| 109 |
+
lexical_score DOUBLE PRECISION
|
| 110 |
+
)
|
| 111 |
+
LANGUAGE plpgsql
|
| 112 |
+
STABLE
|
| 113 |
+
SECURITY DEFINER
|
| 114 |
+
SET search_path = public
|
| 115 |
+
AS $$
|
| 116 |
+
DECLARE
|
| 117 |
+
target_table TEXT;
|
| 118 |
+
BEGIN
|
| 119 |
+
target_table := CASE p_resource_type
|
| 120 |
+
WHEN 'places' THEN 'place_embeddings'
|
| 121 |
+
WHEN 'posts' THEN 'post_embeddings'
|
| 122 |
+
WHEN 'users' THEN 'user_embeddings'
|
| 123 |
+
WHEN 'clubs' THEN 'club_embeddings'
|
| 124 |
+
WHEN 'groups' THEN 'group_embeddings'
|
| 125 |
+
WHEN 'events' THEN 'event_embeddings'
|
| 126 |
+
ELSE NULL
|
| 127 |
+
END;
|
| 128 |
+
IF target_table IS NULL THEN
|
| 129 |
+
RAISE EXCEPTION 'Unsupported search resource type: %', p_resource_type;
|
| 130 |
+
END IF;
|
| 131 |
+
|
| 132 |
+
RETURN QUERY EXECUTE format($query$
|
| 133 |
+
WITH eligible AS NOT MATERIALIZED (
|
| 134 |
+
SELECT e.*
|
| 135 |
+
FROM public.%I e
|
| 136 |
+
WHERE e.is_active = true
|
| 137 |
+
AND COALESCE(($4->>'is_active')::boolean, true) = true
|
| 138 |
+
AND (
|
| 139 |
+
($4 ? 'city') IS FALSE
|
| 140 |
+
OR lower(e.metadata->>'city') = lower($4->>'city')
|
| 141 |
+
)
|
| 142 |
+
AND (
|
| 143 |
+
($4 ? 'state') IS FALSE
|
| 144 |
+
OR lower(e.metadata->>'state') = lower($4->>'state')
|
| 145 |
+
)
|
| 146 |
+
AND (
|
| 147 |
+
($4 ? 'categories') IS FALSE
|
| 148 |
+
OR EXISTS (
|
| 149 |
+
SELECT 1
|
| 150 |
+
FROM jsonb_array_elements_text($4->'categories') AS allowed(value)
|
| 151 |
+
WHERE lower(e.metadata->>'category') = lower(allowed.value)
|
| 152 |
+
)
|
| 153 |
+
)
|
| 154 |
+
AND (
|
| 155 |
+
($4 ? 'tags') IS FALSE
|
| 156 |
+
OR EXISTS (
|
| 157 |
+
SELECT 1
|
| 158 |
+
FROM jsonb_array_elements_text($4->'tags') AS allowed(value)
|
| 159 |
+
WHERE lower(COALESCE(e.metadata->>'tags', ''))
|
| 160 |
+
LIKE ('%%' || lower(allowed.value) || '%%')
|
| 161 |
+
)
|
| 162 |
+
)
|
| 163 |
+
AND (
|
| 164 |
+
($4 ? 'published_from') IS FALSE
|
| 165 |
+
OR public.try_parse_timestamptz(e.metadata->>'published_at')
|
| 166 |
+
>= ($4->>'published_from')::timestamptz
|
| 167 |
+
)
|
| 168 |
+
AND (
|
| 169 |
+
($4 ? 'published_to') IS FALSE
|
| 170 |
+
OR public.try_parse_timestamptz(e.metadata->>'published_at')
|
| 171 |
+
<= ($4->>'published_to')::timestamptz
|
| 172 |
+
)
|
| 173 |
+
AND (
|
| 174 |
+
($4 ? 'event_from') IS FALSE
|
| 175 |
+
OR public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 176 |
+
>= ($4->>'event_from')::timestamptz
|
| 177 |
+
)
|
| 178 |
+
AND (
|
| 179 |
+
($4 ? 'event_to') IS FALSE
|
| 180 |
+
OR public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 181 |
+
<= ($4->>'event_to')::timestamptz
|
| 182 |
+
)
|
| 183 |
+
AND (
|
| 184 |
+
($4 ? 'event_active_at') IS FALSE
|
| 185 |
+
OR (
|
| 186 |
+
public.try_parse_timestamptz(e.metadata->>'start_time') IS NOT NULL
|
| 187 |
+
AND public.try_parse_positive_integer(
|
| 188 |
+
e.metadata->>'duration_minutes'
|
| 189 |
+
) IS NOT NULL
|
| 190 |
+
AND public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 191 |
+
+ make_interval(
|
| 192 |
+
mins => public.try_parse_positive_integer(
|
| 193 |
+
e.metadata->>'duration_minutes'
|
| 194 |
+
)
|
| 195 |
+
)
|
| 196 |
+
> ($4->>'event_active_at')::timestamptz
|
| 197 |
+
)
|
| 198 |
+
)
|
| 199 |
+
AND (
|
| 200 |
+
($4 ? 'is_online') IS FALSE
|
| 201 |
+
OR COALESCE((e.metadata->>'is_online')::boolean, false)
|
| 202 |
+
= ($4->>'is_online')::boolean
|
| 203 |
+
)
|
| 204 |
+
AND (
|
| 205 |
+
($4 ? 'user_roles') IS FALSE
|
| 206 |
+
OR EXISTS (
|
| 207 |
+
SELECT 1
|
| 208 |
+
FROM jsonb_array_elements_text($4->'user_roles') AS allowed(value)
|
| 209 |
+
WHERE lower(e.metadata->>'role') = lower(allowed.value)
|
| 210 |
+
)
|
| 211 |
+
)
|
| 212 |
+
AND (
|
| 213 |
+
($4 ? 'place_ids') IS FALSE
|
| 214 |
+
OR e.metadata->>'place_id' IN (
|
| 215 |
+
SELECT jsonb_array_elements_text($4->'place_ids')
|
| 216 |
+
)
|
| 217 |
+
)
|
| 218 |
+
AND (
|
| 219 |
+
(
|
| 220 |
+
COALESCE((e.metadata->>'is_private')::boolean, false) = false
|
| 221 |
+
AND COALESCE((e.metadata->>'is_public')::boolean, true) = true
|
| 222 |
+
)
|
| 223 |
+
OR (
|
| 224 |
+
$4 ? 'requester_id'
|
| 225 |
+
AND (
|
| 226 |
+
e.metadata->>'creator_id' = $4->>'requester_id'
|
| 227 |
+
OR COALESCE(e.metadata->'authorized_user_ids', '[]'::jsonb)
|
| 228 |
+
? ($4->>'requester_id')
|
| 229 |
+
)
|
| 230 |
+
)
|
| 231 |
+
)
|
| 232 |
+
),
|
| 233 |
+
vector_results AS (
|
| 234 |
+
SELECT
|
| 235 |
+
e.external_id,
|
| 236 |
+
1 - (e.embedding <=> $1) AS semantic_score,
|
| 237 |
+
row_number() OVER (
|
| 238 |
+
ORDER BY e.embedding <=> $1, e.external_id ASC
|
| 239 |
+
) AS vector_rank
|
| 240 |
+
FROM eligible e
|
| 241 |
+
ORDER BY e.embedding <=> $1, e.external_id ASC
|
| 242 |
+
LIMIT GREATEST($3 * 3, 30)
|
| 243 |
+
),
|
| 244 |
+
lexical_results AS (
|
| 245 |
+
SELECT
|
| 246 |
+
e.external_id,
|
| 247 |
+
ts_rank_cd(e.textsearch, query.value) AS lexical_score,
|
| 248 |
+
row_number() OVER (
|
| 249 |
+
ORDER BY ts_rank_cd(e.textsearch, query.value) DESC, e.external_id ASC
|
| 250 |
+
) AS lexical_rank
|
| 251 |
+
FROM eligible e
|
| 252 |
+
CROSS JOIN websearch_to_tsquery('simple', $2) AS query(value)
|
| 253 |
+
WHERE e.textsearch @@ query.value
|
| 254 |
+
ORDER BY lexical_score DESC, e.external_id ASC
|
| 255 |
+
LIMIT GREATEST($3 * 3, 30)
|
| 256 |
+
),
|
| 257 |
+
fused AS (
|
| 258 |
+
SELECT
|
| 259 |
+
COALESCE(v.external_id, l.external_id) AS external_id,
|
| 260 |
+
v.semantic_score,
|
| 261 |
+
l.lexical_score,
|
| 262 |
+
COALESCE(1.0 / (60.0 + v.vector_rank), 0.0)
|
| 263 |
+
+ COALESCE(1.0 / (60.0 + l.lexical_rank), 0.0) AS rrf_score
|
| 264 |
+
FROM vector_results v
|
| 265 |
+
FULL OUTER JOIN lexical_results l USING (external_id)
|
| 266 |
+
)
|
| 267 |
+
SELECT
|
| 268 |
+
e.external_id,
|
| 269 |
+
e.document,
|
| 270 |
+
e.metadata,
|
| 271 |
+
LEAST(
|
| 272 |
+
1.0,
|
| 273 |
+
f.rrf_score / (2.0 / 61.0)
|
| 274 |
+
+ CASE
|
| 275 |
+
WHEN lower(COALESCE(
|
| 276 |
+
e.metadata->>'search_title',
|
| 277 |
+
e.metadata->>'name',
|
| 278 |
+
e.metadata->>'title',
|
| 279 |
+
e.metadata->>'username',
|
| 280 |
+
''
|
| 281 |
+
)) = lower($2) THEN 0.15
|
| 282 |
+
ELSE 0.0
|
| 283 |
+
END
|
| 284 |
+
)::double precision AS score,
|
| 285 |
+
f.semantic_score::double precision,
|
| 286 |
+
f.lexical_score::double precision
|
| 287 |
+
FROM fused f
|
| 288 |
+
JOIN eligible e USING (external_id)
|
| 289 |
+
WHERE (
|
| 290 |
+
(
|
| 291 |
+
($4 ? 'min_semantic_score') IS FALSE
|
| 292 |
+
AND ($4 ? 'min_lexical_score') IS FALSE
|
| 293 |
+
)
|
| 294 |
+
OR (
|
| 295 |
+
($4 ? 'min_semantic_score')
|
| 296 |
+
AND COALESCE(
|
| 297 |
+
f.semantic_score >= ($4->>'min_semantic_score')::double precision,
|
| 298 |
+
FALSE
|
| 299 |
+
)
|
| 300 |
+
)
|
| 301 |
+
OR (
|
| 302 |
+
($4 ? 'min_lexical_score')
|
| 303 |
+
AND COALESCE(
|
| 304 |
+
f.lexical_score >= ($4->>'min_lexical_score')::double precision,
|
| 305 |
+
FALSE
|
| 306 |
+
)
|
| 307 |
+
)
|
| 308 |
+
)
|
| 309 |
+
ORDER BY score DESC, f.semantic_score DESC NULLS LAST, e.external_id ASC
|
| 310 |
+
LIMIT GREATEST($3, 1)
|
| 311 |
+
$query$, target_table)
|
| 312 |
+
USING p_query_embedding, p_query_text, p_match_count, p_filters;
|
| 313 |
+
END;
|
| 314 |
+
$$;
|
| 315 |
+
|
| 316 |
+
REVOKE ALL ON FUNCTION public.search_resource_embeddings(
|
| 317 |
+
text, text, vector, integer, jsonb
|
| 318 |
+
) FROM PUBLIC;
|
| 319 |
+
REVOKE ALL ON FUNCTION public.try_parse_timestamptz(text) FROM PUBLIC;
|
| 320 |
+
REVOKE ALL ON FUNCTION public.try_parse_positive_integer(text) FROM PUBLIC;
|
| 321 |
+
GRANT EXECUTE ON FUNCTION public.search_resource_embeddings(
|
| 322 |
+
text, text, vector, integer, jsonb
|
| 323 |
+
) TO nlp_reader;
|
| 324 |
+
GRANT EXECUTE ON FUNCTION public.try_parse_timestamptz(text) TO nlp_reader;
|
| 325 |
+
GRANT EXECUTE ON FUNCTION public.try_parse_positive_integer(text) TO nlp_reader;
|
| 326 |
+
|
| 327 |
+
DO $postcheck$
|
| 328 |
+
DECLARE
|
| 329 |
+
installed_definition TEXT;
|
| 330 |
+
BEGIN
|
| 331 |
+
SELECT pg_get_functiondef(
|
| 332 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)'::regprocedure
|
| 333 |
+
) INTO installed_definition;
|
| 334 |
+
|
| 335 |
+
IF installed_definition NOT LIKE '%event_active_at%'
|
| 336 |
+
OR installed_definition NOT LIKE '%min_semantic_score%'
|
| 337 |
+
OR installed_definition NOT LIKE '%min_lexical_score%'
|
| 338 |
+
OR installed_definition NOT LIKE '%try_parse_timestamptz%'
|
| 339 |
+
OR installed_definition NOT LIKE '%try_parse_positive_integer%' THEN
|
| 340 |
+
RAISE EXCEPTION 'Postcheck search V2: la definicion instalada esta incompleta';
|
| 341 |
+
END IF;
|
| 342 |
+
|
| 343 |
+
IF NOT has_function_privilege(
|
| 344 |
+
'nlp_reader',
|
| 345 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)',
|
| 346 |
+
'EXECUTE'
|
| 347 |
+
) THEN
|
| 348 |
+
RAISE EXCEPTION 'Postcheck search V2: nlp_reader no conserva EXECUTE';
|
| 349 |
+
END IF;
|
| 350 |
+
|
| 351 |
+
IF NOT has_function_privilege(
|
| 352 |
+
'nlp_reader', 'public.try_parse_timestamptz(text)', 'EXECUTE'
|
| 353 |
+
) OR NOT has_function_privilege(
|
| 354 |
+
'nlp_reader', 'public.try_parse_positive_integer(text)', 'EXECUTE'
|
| 355 |
+
) THEN
|
| 356 |
+
RAISE EXCEPTION 'Postcheck search V2: nlp_reader no puede ejecutar los helpers seguros';
|
| 357 |
+
END IF;
|
| 358 |
+
END;
|
| 359 |
+
$postcheck$;
|
| 360 |
+
|
| 361 |
+
DO $functional_postcheck$
|
| 362 |
+
DECLARE
|
| 363 |
+
zero_embedding VECTOR(300);
|
| 364 |
+
invalid_count BIGINT;
|
| 365 |
+
BEGIN
|
| 366 |
+
zero_embedding := (
|
| 367 |
+
'[' || rtrim(repeat('0,', 300), ',') || ']'
|
| 368 |
+
)::vector;
|
| 369 |
+
|
| 370 |
+
SELECT COUNT(*) INTO invalid_count
|
| 371 |
+
FROM public.search_resource_embeddings(
|
| 372 |
+
'users',
|
| 373 |
+
'__frimeet_verification_no_match_9f7c2__',
|
| 374 |
+
zero_embedding,
|
| 375 |
+
100,
|
| 376 |
+
jsonb_build_object(
|
| 377 |
+
'is_active', TRUE,
|
| 378 |
+
'min_semantic_score', 1.000001,
|
| 379 |
+
'min_lexical_score', 1000000.0
|
| 380 |
+
)
|
| 381 |
+
);
|
| 382 |
+
IF invalid_count <> 0 THEN
|
| 383 |
+
RAISE EXCEPTION 'Postcheck search V2: una consulta bajo umbrales imposibles devolvio % filas', invalid_count;
|
| 384 |
+
END IF;
|
| 385 |
+
|
| 386 |
+
SELECT COUNT(*) INTO invalid_count
|
| 387 |
+
FROM public.search_resource_embeddings(
|
| 388 |
+
'events',
|
| 389 |
+
'__frimeet_event_filter_verification__',
|
| 390 |
+
zero_embedding,
|
| 391 |
+
1000,
|
| 392 |
+
jsonb_build_object(
|
| 393 |
+
'is_active', TRUE,
|
| 394 |
+
'event_active_at', CURRENT_TIMESTAMP
|
| 395 |
+
)
|
| 396 |
+
) result
|
| 397 |
+
WHERE public.try_parse_timestamptz(result.metadata->>'start_time') IS NULL
|
| 398 |
+
OR public.try_parse_positive_integer(
|
| 399 |
+
result.metadata->>'duration_minutes'
|
| 400 |
+
) IS NULL
|
| 401 |
+
OR public.try_parse_timestamptz(result.metadata->>'start_time')
|
| 402 |
+
+ make_interval(
|
| 403 |
+
mins => public.try_parse_positive_integer(
|
| 404 |
+
result.metadata->>'duration_minutes'
|
| 405 |
+
)
|
| 406 |
+
) <= CURRENT_TIMESTAMP;
|
| 407 |
+
IF invalid_count <> 0 THEN
|
| 408 |
+
RAISE EXCEPTION 'Postcheck search V2: se devolvieron % eventos terminados o invalidos', invalid_count;
|
| 409 |
+
END IF;
|
| 410 |
+
END;
|
| 411 |
+
$functional_postcheck$;
|
| 412 |
+
|
| 413 |
+
COMMIT;
|
sql/new_pgvector_schema.sql
CHANGED
|
@@ -424,6 +424,37 @@ AS $$
|
|
| 424 |
WHERE p.external_id = ANY(p_external_ids);
|
| 425 |
$$;
|
| 426 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 427 |
CREATE OR REPLACE FUNCTION search_resource_embeddings(
|
| 428 |
p_resource_type TEXT,
|
| 429 |
p_query_text TEXT,
|
|
@@ -493,24 +524,40 @@ BEGIN
|
|
| 493 |
)
|
| 494 |
AND (
|
| 495 |
($4 ? 'published_from') IS FALSE
|
| 496 |
-
OR
|
| 497 |
>= ($4->>'published_from')::timestamptz
|
| 498 |
)
|
| 499 |
AND (
|
| 500 |
($4 ? 'published_to') IS FALSE
|
| 501 |
-
OR
|
| 502 |
<= ($4->>'published_to')::timestamptz
|
| 503 |
)
|
| 504 |
AND (
|
| 505 |
($4 ? 'event_from') IS FALSE
|
| 506 |
-
OR
|
| 507 |
>= ($4->>'event_from')::timestamptz
|
| 508 |
)
|
| 509 |
AND (
|
| 510 |
($4 ? 'event_to') IS FALSE
|
| 511 |
-
OR
|
| 512 |
<= ($4->>'event_to')::timestamptz
|
| 513 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 514 |
AND (
|
| 515 |
($4 ? 'is_online') IS FALSE
|
| 516 |
OR COALESCE((e.metadata->>'is_online')::boolean, false)
|
|
@@ -601,6 +648,26 @@ BEGIN
|
|
| 601 |
f.lexical_score::double precision
|
| 602 |
FROM fused f
|
| 603 |
JOIN eligible e USING (external_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
ORDER BY score DESC, f.semantic_score DESC NULLS LAST, e.external_id ASC
|
| 605 |
LIMIT GREATEST($3, 1)
|
| 606 |
$query$, target_table)
|
|
@@ -608,6 +675,11 @@ BEGIN
|
|
| 608 |
END;
|
| 609 |
$$;
|
| 610 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 611 |
CREATE OR REPLACE FUNCTION get_resource_content_hashes(
|
| 612 |
p_resource_type TEXT,
|
| 613 |
p_external_ids TEXT[]
|
|
|
|
| 424 |
WHERE p.external_id = ANY(p_external_ids);
|
| 425 |
$$;
|
| 426 |
|
| 427 |
+
CREATE OR REPLACE FUNCTION try_parse_timestamptz(p_value TEXT)
|
| 428 |
+
RETURNS TIMESTAMPTZ
|
| 429 |
+
LANGUAGE plpgsql
|
| 430 |
+
IMMUTABLE
|
| 431 |
+
PARALLEL SAFE
|
| 432 |
+
SET search_path = pg_catalog
|
| 433 |
+
AS $$
|
| 434 |
+
BEGIN
|
| 435 |
+
RETURN NULLIF(btrim(p_value), '')::timestamptz;
|
| 436 |
+
EXCEPTION WHEN OTHERS THEN
|
| 437 |
+
RETURN NULL;
|
| 438 |
+
END;
|
| 439 |
+
$$;
|
| 440 |
+
|
| 441 |
+
CREATE OR REPLACE FUNCTION try_parse_positive_integer(p_value TEXT)
|
| 442 |
+
RETURNS INTEGER
|
| 443 |
+
LANGUAGE plpgsql
|
| 444 |
+
IMMUTABLE
|
| 445 |
+
PARALLEL SAFE
|
| 446 |
+
SET search_path = pg_catalog
|
| 447 |
+
AS $$
|
| 448 |
+
DECLARE
|
| 449 |
+
parsed INTEGER;
|
| 450 |
+
BEGIN
|
| 451 |
+
parsed := NULLIF(btrim(p_value), '')::integer;
|
| 452 |
+
RETURN CASE WHEN parsed > 0 THEN parsed ELSE NULL END;
|
| 453 |
+
EXCEPTION WHEN OTHERS THEN
|
| 454 |
+
RETURN NULL;
|
| 455 |
+
END;
|
| 456 |
+
$$;
|
| 457 |
+
|
| 458 |
CREATE OR REPLACE FUNCTION search_resource_embeddings(
|
| 459 |
p_resource_type TEXT,
|
| 460 |
p_query_text TEXT,
|
|
|
|
| 524 |
)
|
| 525 |
AND (
|
| 526 |
($4 ? 'published_from') IS FALSE
|
| 527 |
+
OR public.try_parse_timestamptz(e.metadata->>'published_at')
|
| 528 |
>= ($4->>'published_from')::timestamptz
|
| 529 |
)
|
| 530 |
AND (
|
| 531 |
($4 ? 'published_to') IS FALSE
|
| 532 |
+
OR public.try_parse_timestamptz(e.metadata->>'published_at')
|
| 533 |
<= ($4->>'published_to')::timestamptz
|
| 534 |
)
|
| 535 |
AND (
|
| 536 |
($4 ? 'event_from') IS FALSE
|
| 537 |
+
OR public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 538 |
>= ($4->>'event_from')::timestamptz
|
| 539 |
)
|
| 540 |
AND (
|
| 541 |
($4 ? 'event_to') IS FALSE
|
| 542 |
+
OR public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 543 |
<= ($4->>'event_to')::timestamptz
|
| 544 |
)
|
| 545 |
+
AND (
|
| 546 |
+
($4 ? 'event_active_at') IS FALSE
|
| 547 |
+
OR (
|
| 548 |
+
public.try_parse_timestamptz(e.metadata->>'start_time') IS NOT NULL
|
| 549 |
+
AND public.try_parse_positive_integer(
|
| 550 |
+
e.metadata->>'duration_minutes'
|
| 551 |
+
) IS NOT NULL
|
| 552 |
+
AND public.try_parse_timestamptz(e.metadata->>'start_time')
|
| 553 |
+
+ make_interval(
|
| 554 |
+
mins => public.try_parse_positive_integer(
|
| 555 |
+
e.metadata->>'duration_minutes'
|
| 556 |
+
)
|
| 557 |
+
)
|
| 558 |
+
> ($4->>'event_active_at')::timestamptz
|
| 559 |
+
)
|
| 560 |
+
)
|
| 561 |
AND (
|
| 562 |
($4 ? 'is_online') IS FALSE
|
| 563 |
OR COALESCE((e.metadata->>'is_online')::boolean, false)
|
|
|
|
| 648 |
f.lexical_score::double precision
|
| 649 |
FROM fused f
|
| 650 |
JOIN eligible e USING (external_id)
|
| 651 |
+
WHERE (
|
| 652 |
+
(
|
| 653 |
+
($4 ? 'min_semantic_score') IS FALSE
|
| 654 |
+
AND ($4 ? 'min_lexical_score') IS FALSE
|
| 655 |
+
)
|
| 656 |
+
OR (
|
| 657 |
+
($4 ? 'min_semantic_score')
|
| 658 |
+
AND COALESCE(
|
| 659 |
+
f.semantic_score >= ($4->>'min_semantic_score')::double precision,
|
| 660 |
+
FALSE
|
| 661 |
+
)
|
| 662 |
+
)
|
| 663 |
+
OR (
|
| 664 |
+
($4 ? 'min_lexical_score')
|
| 665 |
+
AND COALESCE(
|
| 666 |
+
f.lexical_score >= ($4->>'min_lexical_score')::double precision,
|
| 667 |
+
FALSE
|
| 668 |
+
)
|
| 669 |
+
)
|
| 670 |
+
)
|
| 671 |
ORDER BY score DESC, f.semantic_score DESC NULLS LAST, e.external_id ASC
|
| 672 |
LIMIT GREATEST($3, 1)
|
| 673 |
$query$, target_table)
|
|
|
|
| 675 |
END;
|
| 676 |
$$;
|
| 677 |
|
| 678 |
+
REVOKE ALL ON FUNCTION public.try_parse_timestamptz(text) FROM PUBLIC;
|
| 679 |
+
REVOKE ALL ON FUNCTION public.try_parse_positive_integer(text) FROM PUBLIC;
|
| 680 |
+
GRANT EXECUTE ON FUNCTION public.try_parse_timestamptz(text) TO nlp_reader;
|
| 681 |
+
GRANT EXECUTE ON FUNCTION public.try_parse_positive_integer(text) TO nlp_reader;
|
| 682 |
+
|
| 683 |
CREATE OR REPLACE FUNCTION get_resource_content_hashes(
|
| 684 |
p_resource_type TEXT,
|
| 685 |
p_external_ids TEXT[]
|
sql/verify_global_search_candidate_filters.sql
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- Verificador de solo lectura para Busqueda Global V2.
|
| 2 |
+
-- Ejecutar despues de sql/migrations/20260712_01_global_search_candidate_filters.sql.
|
| 3 |
+
|
| 4 |
+
BEGIN;
|
| 5 |
+
SET TRANSACTION READ ONLY;
|
| 6 |
+
|
| 7 |
+
WITH function_contract AS (
|
| 8 |
+
SELECT
|
| 9 |
+
to_regprocedure(
|
| 10 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)'
|
| 11 |
+
) AS function_oid
|
| 12 |
+
), definition AS (
|
| 13 |
+
SELECT pg_get_functiondef(function_oid) AS body
|
| 14 |
+
FROM function_contract
|
| 15 |
+
WHERE function_oid IS NOT NULL
|
| 16 |
+
)
|
| 17 |
+
SELECT
|
| 18 |
+
(SELECT function_oid IS NOT NULL FROM function_contract) AS function_exists,
|
| 19 |
+
COALESCE((SELECT body LIKE '%event_active_at%' FROM definition), FALSE)
|
| 20 |
+
AS has_event_active_filter,
|
| 21 |
+
COALESCE((SELECT body LIKE '%min_semantic_score%' FROM definition), FALSE)
|
| 22 |
+
AS has_semantic_threshold,
|
| 23 |
+
COALESCE((SELECT body LIKE '%min_lexical_score%' FROM definition), FALSE)
|
| 24 |
+
AS has_lexical_threshold,
|
| 25 |
+
COALESCE((SELECT body LIKE '%try_parse_timestamptz%' FROM definition), FALSE)
|
| 26 |
+
AS has_safe_timestamp_parser,
|
| 27 |
+
to_regprocedure('public.try_parse_positive_integer(text)') IS NOT NULL
|
| 28 |
+
AS has_safe_duration_parser,
|
| 29 |
+
has_function_privilege(
|
| 30 |
+
'nlp_reader',
|
| 31 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)',
|
| 32 |
+
'EXECUTE'
|
| 33 |
+
) AS reader_can_execute;
|
| 34 |
+
|
| 35 |
+
WITH zero_embedding AS (
|
| 36 |
+
SELECT (
|
| 37 |
+
'[' || rtrim(repeat('0,', 300), ',') || ']'
|
| 38 |
+
)::vector AS value
|
| 39 |
+
), negative_results AS (
|
| 40 |
+
SELECT result.external_id
|
| 41 |
+
FROM zero_embedding embedding
|
| 42 |
+
CROSS JOIN LATERAL public.search_resource_embeddings(
|
| 43 |
+
'users',
|
| 44 |
+
'__frimeet_verification_no_match_9f7c2__',
|
| 45 |
+
embedding.value,
|
| 46 |
+
100,
|
| 47 |
+
jsonb_build_object(
|
| 48 |
+
'is_active', TRUE,
|
| 49 |
+
'min_semantic_score', 1.000001,
|
| 50 |
+
'min_lexical_score', 1000000.0
|
| 51 |
+
)
|
| 52 |
+
) result
|
| 53 |
+
)
|
| 54 |
+
SELECT COUNT(*) AS negative_query_results
|
| 55 |
+
FROM negative_results;
|
| 56 |
+
|
| 57 |
+
WITH zero_embedding AS (
|
| 58 |
+
SELECT (
|
| 59 |
+
'[' || rtrim(repeat('0,', 300), ',') || ']'
|
| 60 |
+
)::vector AS value
|
| 61 |
+
), event_results AS (
|
| 62 |
+
SELECT result.metadata
|
| 63 |
+
FROM zero_embedding embedding
|
| 64 |
+
CROSS JOIN LATERAL public.search_resource_embeddings(
|
| 65 |
+
'events',
|
| 66 |
+
'__frimeet_event_filter_verification__',
|
| 67 |
+
embedding.value,
|
| 68 |
+
1000,
|
| 69 |
+
jsonb_build_object(
|
| 70 |
+
'is_active', TRUE,
|
| 71 |
+
'event_active_at', CURRENT_TIMESTAMP
|
| 72 |
+
)
|
| 73 |
+
) result
|
| 74 |
+
)
|
| 75 |
+
SELECT COUNT(*) AS ended_or_invalid_events_returned
|
| 76 |
+
FROM event_results
|
| 77 |
+
WHERE public.try_parse_timestamptz(metadata->>'start_time') IS NULL
|
| 78 |
+
OR public.try_parse_positive_integer(metadata->>'duration_minutes') IS NULL
|
| 79 |
+
OR public.try_parse_timestamptz(metadata->>'start_time')
|
| 80 |
+
+ make_interval(
|
| 81 |
+
mins => public.try_parse_positive_integer(
|
| 82 |
+
metadata->>'duration_minutes'
|
| 83 |
+
)
|
| 84 |
+
)
|
| 85 |
+
<= CURRENT_TIMESTAMP;
|
| 86 |
+
|
| 87 |
+
DO $assertions$
|
| 88 |
+
DECLARE
|
| 89 |
+
function_body TEXT;
|
| 90 |
+
zero_embedding VECTOR(300);
|
| 91 |
+
invalid_count BIGINT;
|
| 92 |
+
BEGIN
|
| 93 |
+
IF to_regprocedure(
|
| 94 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)'
|
| 95 |
+
) IS NULL THEN
|
| 96 |
+
RAISE EXCEPTION 'Verify search V2: falta search_resource_embeddings';
|
| 97 |
+
END IF;
|
| 98 |
+
IF to_regprocedure('public.try_parse_timestamptz(text)') IS NULL
|
| 99 |
+
OR to_regprocedure('public.try_parse_positive_integer(text)') IS NULL THEN
|
| 100 |
+
RAISE EXCEPTION 'Verify search V2: faltan helpers de parseo seguro';
|
| 101 |
+
END IF;
|
| 102 |
+
|
| 103 |
+
SELECT pg_get_functiondef(
|
| 104 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)'::regprocedure
|
| 105 |
+
) INTO function_body;
|
| 106 |
+
IF function_body NOT LIKE '%event_active_at%'
|
| 107 |
+
OR function_body NOT LIKE '%min_semantic_score%'
|
| 108 |
+
OR function_body NOT LIKE '%min_lexical_score%'
|
| 109 |
+
OR function_body NOT LIKE '%try_parse_timestamptz%'
|
| 110 |
+
OR function_body NOT LIKE '%try_parse_positive_integer%' THEN
|
| 111 |
+
RAISE EXCEPTION 'Verify search V2: contrato funcional incompleto';
|
| 112 |
+
END IF;
|
| 113 |
+
|
| 114 |
+
IF NOT has_function_privilege(
|
| 115 |
+
'nlp_reader',
|
| 116 |
+
'public.search_resource_embeddings(text,text,vector,integer,jsonb)',
|
| 117 |
+
'EXECUTE'
|
| 118 |
+
) THEN
|
| 119 |
+
RAISE EXCEPTION 'Verify search V2: nlp_reader no puede ejecutar la busqueda';
|
| 120 |
+
END IF;
|
| 121 |
+
|
| 122 |
+
SELECT COUNT(*) INTO invalid_count
|
| 123 |
+
FROM public.event_embeddings
|
| 124 |
+
WHERE is_active = TRUE
|
| 125 |
+
AND (
|
| 126 |
+
public.try_parse_timestamptz(metadata->>'start_time') IS NULL
|
| 127 |
+
OR public.try_parse_positive_integer(
|
| 128 |
+
metadata->>'duration_minutes'
|
| 129 |
+
) IS NULL
|
| 130 |
+
);
|
| 131 |
+
IF invalid_count <> 0 THEN
|
| 132 |
+
RAISE EXCEPTION USING
|
| 133 |
+
MESSAGE = format(
|
| 134 |
+
'Verify search V2: existen %s eventos activos con metadata temporal invalida',
|
| 135 |
+
invalid_count
|
| 136 |
+
),
|
| 137 |
+
HINT = 'Ejecute nuevamente sync_search_embeddings para events antes de activar search.';
|
| 138 |
+
END IF;
|
| 139 |
+
|
| 140 |
+
zero_embedding := (
|
| 141 |
+
'[' || rtrim(repeat('0,', 300), ',') || ']'
|
| 142 |
+
)::vector;
|
| 143 |
+
SELECT COUNT(*) INTO invalid_count
|
| 144 |
+
FROM public.search_resource_embeddings(
|
| 145 |
+
'users',
|
| 146 |
+
'__frimeet_verification_no_match_9f7c2__',
|
| 147 |
+
zero_embedding,
|
| 148 |
+
100,
|
| 149 |
+
jsonb_build_object(
|
| 150 |
+
'is_active', TRUE,
|
| 151 |
+
'min_semantic_score', 1.000001,
|
| 152 |
+
'min_lexical_score', 1000000.0
|
| 153 |
+
)
|
| 154 |
+
);
|
| 155 |
+
IF invalid_count <> 0 THEN
|
| 156 |
+
RAISE EXCEPTION 'Verify search V2: consulta negativa devolvio % filas', invalid_count;
|
| 157 |
+
END IF;
|
| 158 |
+
|
| 159 |
+
SELECT COUNT(*) INTO invalid_count
|
| 160 |
+
FROM public.search_resource_embeddings(
|
| 161 |
+
'events',
|
| 162 |
+
'__frimeet_event_filter_verification__',
|
| 163 |
+
zero_embedding,
|
| 164 |
+
1000,
|
| 165 |
+
jsonb_build_object(
|
| 166 |
+
'is_active', TRUE,
|
| 167 |
+
'event_active_at', CURRENT_TIMESTAMP
|
| 168 |
+
)
|
| 169 |
+
) result
|
| 170 |
+
WHERE public.try_parse_timestamptz(result.metadata->>'start_time') IS NULL
|
| 171 |
+
OR public.try_parse_positive_integer(
|
| 172 |
+
result.metadata->>'duration_minutes'
|
| 173 |
+
) IS NULL
|
| 174 |
+
OR public.try_parse_timestamptz(result.metadata->>'start_time')
|
| 175 |
+
+ make_interval(
|
| 176 |
+
mins => public.try_parse_positive_integer(
|
| 177 |
+
result.metadata->>'duration_minutes'
|
| 178 |
+
)
|
| 179 |
+
) <= CURRENT_TIMESTAMP;
|
| 180 |
+
IF invalid_count <> 0 THEN
|
| 181 |
+
RAISE EXCEPTION 'Verify search V2: se devolvieron % eventos invalidos', invalid_count;
|
| 182 |
+
END IF;
|
| 183 |
+
END;
|
| 184 |
+
$assertions$;
|
| 185 |
+
|
| 186 |
+
COMMIT;
|
tests/conftest.py
CHANGED
|
@@ -7,3 +7,8 @@ os.environ["EMBEDDING_MODEL"] = "mock-embedding"
|
|
| 7 |
os.environ["EMBEDDING_VERSION"] = "test-v1"
|
| 8 |
os.environ["GROQ_API_KEY"] = ""
|
| 9 |
os.environ["SEARCH_INTERNAL_TOKEN"] = "test-search-token"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
os.environ["EMBEDDING_VERSION"] = "test-v1"
|
| 8 |
os.environ["GROQ_API_KEY"] = ""
|
| 9 |
os.environ["SEARCH_INTERNAL_TOKEN"] = "test-search-token"
|
| 10 |
+
os.environ["NLP_SERVICE_TOKEN"] = "test-nlp-service-token"
|
| 11 |
+
os.environ["PUBLIC_GLOBAL_SEARCH_ENABLED"] = "true"
|
| 12 |
+
os.environ["GLOBAL_SEARCH_MIN_SEMANTIC_SCORE"] = "0.30"
|
| 13 |
+
os.environ["GLOBAL_SEARCH_MIN_LEXICAL_SCORE"] = "0.05"
|
| 14 |
+
os.environ["GLOBAL_SEARCH_THRESHOLD_POLICY_VERSION"] = "test-search-policy-v1"
|
tests/test_feed_settings.py
CHANGED
|
@@ -22,3 +22,37 @@ def test_production_rejects_mock_vector_store_for_feed() -> None:
|
|
| 22 |
MAIN_API_INTERNAL_TOKEN="main-token",
|
| 23 |
VECTOR_STORE_PROVIDER="mock",
|
| 24 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
MAIN_API_INTERNAL_TOKEN="main-token",
|
| 23 |
VECTOR_STORE_PROVIDER="mock",
|
| 24 |
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def test_search_threshold_overrides_accept_complete_resource_policy() -> None:
|
| 28 |
+
settings = Settings(
|
| 29 |
+
_env_file=None,
|
| 30 |
+
ENV="local",
|
| 31 |
+
NLP_SERVICE_TOKEN="nlp-token",
|
| 32 |
+
GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON={
|
| 33 |
+
"posts": {"semantic_min": 0.42, "lexical_min": 0.08}
|
| 34 |
+
},
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
assert settings.global_search_resource_thresholds["posts"] == {
|
| 38 |
+
"semantic_min": 0.42,
|
| 39 |
+
"lexical_min": 0.08,
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@pytest.mark.parametrize(
|
| 44 |
+
"overrides",
|
| 45 |
+
[
|
| 46 |
+
{"posts": {"semantic_min": 0.42}},
|
| 47 |
+
{"unknown": {"semantic_min": 0.42, "lexical_min": 0.08}},
|
| 48 |
+
{"posts": {"semantic_min": 1.1, "lexical_min": 0.08}},
|
| 49 |
+
],
|
| 50 |
+
)
|
| 51 |
+
def test_search_threshold_overrides_reject_invalid_policy(overrides: dict) -> None:
|
| 52 |
+
with pytest.raises(ValidationError):
|
| 53 |
+
Settings(
|
| 54 |
+
_env_file=None,
|
| 55 |
+
ENV="local",
|
| 56 |
+
NLP_SERVICE_TOKEN="nlp-token",
|
| 57 |
+
GLOBAL_SEARCH_RESOURCE_THRESHOLDS_JSON=overrides,
|
| 58 |
+
)
|
tests/test_internal_search_candidates.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
|
| 3 |
+
from fastapi.testclient import TestClient
|
| 4 |
+
|
| 5 |
+
from app.main import create_app
|
| 6 |
+
from app.modules.search.api.dependencies import get_search_candidates_use_case
|
| 7 |
+
from app.shared.config.settings import get_settings
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def _payload() -> dict[str, object]:
|
| 11 |
+
return {
|
| 12 |
+
"query": "tester1",
|
| 13 |
+
"resource_types": ["users"],
|
| 14 |
+
"candidate_limit_per_type": 5,
|
| 15 |
+
"top_limit": 5,
|
| 16 |
+
"cursors": {},
|
| 17 |
+
"as_of": "2026-07-12T12:00:00Z",
|
| 18 |
+
"filters": {},
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def test_internal_candidates_requires_nlp_service_token() -> None:
|
| 23 |
+
client = TestClient(create_app())
|
| 24 |
+
|
| 25 |
+
missing = client.post("/internal/search/candidates", json=_payload())
|
| 26 |
+
wrong = client.post(
|
| 27 |
+
"/internal/search/candidates",
|
| 28 |
+
headers={"Authorization": "Bearer wrong"},
|
| 29 |
+
json=_payload(),
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
assert missing.status_code == 401
|
| 33 |
+
assert wrong.status_code == 401
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def test_internal_candidates_returns_only_ids_and_scores() -> None:
|
| 37 |
+
client = TestClient(create_app())
|
| 38 |
+
|
| 39 |
+
response = client.post(
|
| 40 |
+
"/internal/search/candidates",
|
| 41 |
+
headers={"Authorization": "Bearer test-nlp-service-token"},
|
| 42 |
+
json=_payload(),
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
assert response.status_code == 200
|
| 46 |
+
payload = response.json()
|
| 47 |
+
assert set(payload) == {"query", "sections", "pagination", "metadata"}
|
| 48 |
+
assert payload["metadata"] == {
|
| 49 |
+
"strategy": "hybrid_thresholded_candidates_v2",
|
| 50 |
+
"failed_resources": {},
|
| 51 |
+
"threshold_policy_version": "test-search-policy-v1",
|
| 52 |
+
}
|
| 53 |
+
candidate = payload["sections"]["users"][0]
|
| 54 |
+
assert set(candidate) == {
|
| 55 |
+
"id",
|
| 56 |
+
"resource_type",
|
| 57 |
+
"score",
|
| 58 |
+
"semantic_score",
|
| 59 |
+
"lexical_score",
|
| 60 |
+
}
|
| 61 |
+
assert "title" not in candidate
|
| 62 |
+
assert "metadata" not in candidate
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def test_internal_cursor_is_bound_to_as_of_and_policy_context() -> None:
|
| 66 |
+
client = TestClient(create_app())
|
| 67 |
+
headers = {"Authorization": "Bearer test-nlp-service-token"}
|
| 68 |
+
first_payload = {
|
| 69 |
+
"query": "lugar tranquilo",
|
| 70 |
+
"resource_types": ["places"],
|
| 71 |
+
"candidate_limit_per_type": 1,
|
| 72 |
+
"top_limit": 1,
|
| 73 |
+
"cursors": {},
|
| 74 |
+
"as_of": "2026-07-12T12:00:00Z",
|
| 75 |
+
"filters": {},
|
| 76 |
+
}
|
| 77 |
+
first = client.post(
|
| 78 |
+
"/internal/search/candidates", headers=headers, json=first_payload
|
| 79 |
+
)
|
| 80 |
+
assert first.status_code == 200
|
| 81 |
+
cursor = first.json()["pagination"]["places"]["next_cursor"]
|
| 82 |
+
assert cursor
|
| 83 |
+
|
| 84 |
+
changed = dict(first_payload)
|
| 85 |
+
changed["as_of"] = "2026-07-12T12:01:00Z"
|
| 86 |
+
changed["cursors"] = {"places": cursor}
|
| 87 |
+
response = client.post(
|
| 88 |
+
"/internal/search/candidates", headers=headers, json=changed
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
assert response.status_code == 422
|
| 92 |
+
assert response.json()["detail"] == "cursor belongs to another search context"
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def test_internal_candidates_returns_503_when_request_budget_expires() -> None:
|
| 96 |
+
class SlowUseCase:
|
| 97 |
+
async def execute(self, **kwargs: object) -> None:
|
| 98 |
+
await asyncio.sleep(0.05)
|
| 99 |
+
|
| 100 |
+
app = create_app()
|
| 101 |
+
app.dependency_overrides[get_search_candidates_use_case] = lambda: SlowUseCase()
|
| 102 |
+
settings = get_settings()
|
| 103 |
+
original_timeout = settings.request_timeout_seconds
|
| 104 |
+
settings.request_timeout_seconds = 0.01 # type: ignore[assignment]
|
| 105 |
+
try:
|
| 106 |
+
response = TestClient(app).post(
|
| 107 |
+
"/internal/search/candidates",
|
| 108 |
+
headers={"Authorization": "Bearer test-nlp-service-token"},
|
| 109 |
+
json=_payload(),
|
| 110 |
+
)
|
| 111 |
+
finally:
|
| 112 |
+
settings.request_timeout_seconds = original_timeout
|
| 113 |
+
|
| 114 |
+
assert response.status_code == 503
|
| 115 |
+
assert response.json()["detail"] == "Internal search timed out"
|
tests/test_pgvector_readiness.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from app.shared.vector_store.aws_pgvector import _read_contract_is_ready
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def _functions(search_v2: bool) -> dict[str, dict[str, object]]:
|
| 5 |
+
return {
|
| 6 |
+
"match_places": {"exists": True, "executable": True},
|
| 7 |
+
"match_posts": {"exists": True, "executable": True},
|
| 8 |
+
"search_resource_embeddings": {
|
| 9 |
+
"exists": True,
|
| 10 |
+
"executable": True,
|
| 11 |
+
"candidate_filters_v2": search_v2,
|
| 12 |
+
},
|
| 13 |
+
"get_post_feed_features": {"exists": True, "executable": True},
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def test_readiness_rejects_legacy_search_function() -> None:
|
| 18 |
+
assert _read_contract_is_ready(True, _functions(search_v2=False)) is False
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_readiness_accepts_search_candidate_v2_contract() -> None:
|
| 22 |
+
assert _read_contract_is_ready(True, _functions(search_v2=True)) is True
|
tests/test_pgvector_search_provider.py
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
|
|
|
|
|
| 1 |
import pytest
|
| 2 |
|
| 3 |
from app.modules.search.domain.filters import SearchCriteria
|
| 4 |
from app.modules.search.domain.models import SearchResourceType
|
| 5 |
from app.modules.search.infrastructure.pgvector_provider import (
|
|
|
|
| 6 |
PgvectorPlaceSearchProvider,
|
| 7 |
)
|
|
|
|
| 8 |
from app.shared.vector_store.models import VectorMatch
|
| 9 |
|
| 10 |
|
|
@@ -66,3 +70,63 @@ async def test_places_use_same_cosine_match_function_as_recommendations() -> Non
|
|
| 66 |
assert hits[0].score == 0.75
|
| 67 |
assert hits[0].semantic_score == 0.75
|
| 68 |
assert hits[0].lexical_score is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datetime import UTC, datetime
|
| 2 |
+
|
| 3 |
import pytest
|
| 4 |
|
| 5 |
from app.modules.search.domain.filters import SearchCriteria
|
| 6 |
from app.modules.search.domain.models import SearchResourceType
|
| 7 |
from app.modules.search.infrastructure.pgvector_provider import (
|
| 8 |
+
PgvectorHybridSearchProvider,
|
| 9 |
PgvectorPlaceSearchProvider,
|
| 10 |
)
|
| 11 |
+
from app.modules.search.domain.relevance import SearchRelevancePolicy
|
| 12 |
from app.shared.vector_store.models import VectorMatch
|
| 13 |
|
| 14 |
|
|
|
|
| 70 |
assert hits[0].score == 0.75
|
| 71 |
assert hits[0].semantic_score == 0.75
|
| 72 |
assert hits[0].lexical_score is None
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class RecordingHybridVectorClient:
|
| 76 |
+
def __init__(self) -> None:
|
| 77 |
+
self.calls: list[dict[str, object]] = []
|
| 78 |
+
|
| 79 |
+
async def search_resource_embeddings(self, **kwargs: object) -> list[VectorMatch]:
|
| 80 |
+
self.calls.append(dict(kwargs))
|
| 81 |
+
return [
|
| 82 |
+
VectorMatch(
|
| 83 |
+
id="rejected",
|
| 84 |
+
score=0.9,
|
| 85 |
+
semantic_score=0.39,
|
| 86 |
+
lexical_score=0.09,
|
| 87 |
+
metadata={
|
| 88 |
+
"start_time": "2026-07-12T11:30:00Z",
|
| 89 |
+
"duration_minutes": 120,
|
| 90 |
+
},
|
| 91 |
+
),
|
| 92 |
+
VectorMatch(
|
| 93 |
+
id="lexical",
|
| 94 |
+
score=0.8,
|
| 95 |
+
semantic_score=0.20,
|
| 96 |
+
lexical_score=0.10,
|
| 97 |
+
metadata={
|
| 98 |
+
"start_time": "2026-07-12T11:30:00Z",
|
| 99 |
+
"duration_minutes": 120,
|
| 100 |
+
},
|
| 101 |
+
),
|
| 102 |
+
]
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
@pytest.mark.asyncio
|
| 106 |
+
async def test_hybrid_provider_pushes_event_and_threshold_filters_to_sql() -> None:
|
| 107 |
+
vector_client = RecordingHybridVectorClient()
|
| 108 |
+
policy = SearchRelevancePolicy.uniform(semantic_min=0.40, lexical_min=0.10)
|
| 109 |
+
provider = PgvectorHybridSearchProvider( # type: ignore[arg-type]
|
| 110 |
+
SearchResourceType.EVENTS,
|
| 111 |
+
vector_client,
|
| 112 |
+
policy,
|
| 113 |
+
)
|
| 114 |
+
active_at = datetime(2026, 7, 12, 12, 0, tzinfo=UTC)
|
| 115 |
+
|
| 116 |
+
hits = await provider.search(
|
| 117 |
+
query="evento",
|
| 118 |
+
embedding=[0.1, 0.2, 0.3],
|
| 119 |
+
limit=5,
|
| 120 |
+
offset=0,
|
| 121 |
+
requester_id=None,
|
| 122 |
+
criteria=SearchCriteria(event_active_at=active_at),
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
assert [hit.id for hit in hits] == ["lexical"]
|
| 126 |
+
filters = vector_client.calls[0]["filters"]
|
| 127 |
+
assert filters == {
|
| 128 |
+
"is_active": True,
|
| 129 |
+
"min_semantic_score": 0.40,
|
| 130 |
+
"min_lexical_score": 0.10,
|
| 131 |
+
"event_active_at": active_at.isoformat(),
|
| 132 |
+
}
|
tests/test_rate_limit.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from types import SimpleNamespace
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
from fastapi import HTTPException
|
| 5 |
+
from starlette.requests import Request
|
| 6 |
+
|
| 7 |
+
from app.shared.security import rate_limit
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def _request(path: str = "/internal/search/candidates") -> Request:
|
| 11 |
+
return Request(
|
| 12 |
+
{
|
| 13 |
+
"type": "http",
|
| 14 |
+
"http_version": "1.1",
|
| 15 |
+
"method": "POST",
|
| 16 |
+
"scheme": "http",
|
| 17 |
+
"path": path,
|
| 18 |
+
"raw_path": path.encode(),
|
| 19 |
+
"query_string": b"",
|
| 20 |
+
"headers": [],
|
| 21 |
+
"client": ("127.0.0.1", 12345),
|
| 22 |
+
"server": ("testserver", 80),
|
| 23 |
+
}
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@pytest.mark.asyncio
|
| 28 |
+
async def test_internal_rate_limit_rejects_requests_over_the_window(monkeypatch) -> None:
|
| 29 |
+
settings = SimpleNamespace(
|
| 30 |
+
rate_limit_window_seconds=60,
|
| 31 |
+
rate_limit_requests_per_window=10,
|
| 32 |
+
internal_rate_limit_requests_per_window=1,
|
| 33 |
+
)
|
| 34 |
+
monkeypatch.setattr(rate_limit, "get_settings", lambda: settings)
|
| 35 |
+
rate_limit.reset_rate_limit_state()
|
| 36 |
+
|
| 37 |
+
await rate_limit.rate_limit_placeholder(_request())
|
| 38 |
+
with pytest.raises(HTTPException) as captured:
|
| 39 |
+
await rate_limit.rate_limit_placeholder(_request())
|
| 40 |
+
|
| 41 |
+
assert captured.value.status_code == 429
|
| 42 |
+
assert captured.value.headers["Retry-After"]
|
| 43 |
+
rate_limit.reset_rate_limit_state()
|
tests/test_search_architecture.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def _imports(path: Path) -> list[str]:
|
| 6 |
+
tree = ast.parse(path.read_text(encoding="utf-8"))
|
| 7 |
+
result: list[str] = []
|
| 8 |
+
for node in ast.walk(tree):
|
| 9 |
+
if isinstance(node, ast.ImportFrom) and node.module:
|
| 10 |
+
result.append(node.module)
|
| 11 |
+
elif isinstance(node, ast.Import):
|
| 12 |
+
result.extend(alias.name for alias in node.names)
|
| 13 |
+
return result
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def test_search_domain_and_application_keep_clean_boundaries() -> None:
|
| 17 |
+
root = Path("app/modules/search")
|
| 18 |
+
violations: list[str] = []
|
| 19 |
+
for path in (root / "domain").rglob("*.py"):
|
| 20 |
+
for imported in _imports(path):
|
| 21 |
+
if imported.startswith("app.") and not imported.startswith(
|
| 22 |
+
"app.modules.search.domain"
|
| 23 |
+
):
|
| 24 |
+
violations.append(f"{path}: {imported}")
|
| 25 |
+
for path in (root / "application").rglob("*.py"):
|
| 26 |
+
for imported in _imports(path):
|
| 27 |
+
if imported.startswith("app.") and not imported.startswith(
|
| 28 |
+
(
|
| 29 |
+
"app.modules.search.domain",
|
| 30 |
+
"app.modules.search.application",
|
| 31 |
+
)
|
| 32 |
+
):
|
| 33 |
+
violations.append(f"{path}: {imported}")
|
| 34 |
+
assert violations == []
|
tests/test_search_candidates_use_case.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datetime import UTC, datetime
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from app.modules.search.application.use_cases.search_candidates import (
|
| 6 |
+
SearchCandidatesUseCase,
|
| 7 |
+
)
|
| 8 |
+
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class StubEmbeddingProvider:
|
| 12 |
+
def embed_text(self, text: str) -> list[float]:
|
| 13 |
+
assert text
|
| 14 |
+
return [1.0, 0.0]
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class StubSearchProvider:
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
resource_type: SearchResourceType,
|
| 21 |
+
hits: list[SearchHit] | None = None,
|
| 22 |
+
error: Exception | None = None,
|
| 23 |
+
) -> None:
|
| 24 |
+
self.resource_type = resource_type
|
| 25 |
+
self._hits = hits or []
|
| 26 |
+
self._error = error
|
| 27 |
+
|
| 28 |
+
async def search(self, **kwargs: object) -> list[SearchHit]:
|
| 29 |
+
if self._error is not None:
|
| 30 |
+
raise self._error
|
| 31 |
+
offset = int(kwargs["offset"])
|
| 32 |
+
limit = int(kwargs["limit"])
|
| 33 |
+
return self._hits[offset : offset + limit]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _hit(identifier: str, score: float) -> SearchHit:
|
| 37 |
+
return SearchHit(
|
| 38 |
+
id=identifier,
|
| 39 |
+
resource_type=SearchResourceType.USERS,
|
| 40 |
+
title=identifier,
|
| 41 |
+
score=score,
|
| 42 |
+
semantic_score=score,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@pytest.mark.asyncio
|
| 47 |
+
@pytest.mark.parametrize(
|
| 48 |
+
("hits", "limit", "expected_ids", "has_more"),
|
| 49 |
+
[
|
| 50 |
+
([], 2, [], False),
|
| 51 |
+
([_hit("one", 0.9)], 2, ["one"], False),
|
| 52 |
+
(
|
| 53 |
+
[_hit("one", 0.9), _hit("two", 0.8), _hit("three", 0.7)],
|
| 54 |
+
2,
|
| 55 |
+
["one", "two"],
|
| 56 |
+
True,
|
| 57 |
+
),
|
| 58 |
+
],
|
| 59 |
+
)
|
| 60 |
+
async def test_candidates_returns_zero_one_or_bounded_results_without_fill(
|
| 61 |
+
hits: list[SearchHit],
|
| 62 |
+
limit: int,
|
| 63 |
+
expected_ids: list[str],
|
| 64 |
+
has_more: bool,
|
| 65 |
+
) -> None:
|
| 66 |
+
use_case = SearchCandidatesUseCase(
|
| 67 |
+
embedding_provider=StubEmbeddingProvider(),
|
| 68 |
+
providers=[StubSearchProvider(SearchResourceType.USERS, hits)],
|
| 69 |
+
policy_version="test-v1",
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
result = await use_case.execute(
|
| 73 |
+
query="consulta",
|
| 74 |
+
resource_types=(SearchResourceType.USERS,),
|
| 75 |
+
candidate_limit_per_type=limit,
|
| 76 |
+
as_of=datetime(2026, 7, 12, tzinfo=UTC),
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
assert [item.id for item in result.sections[SearchResourceType.USERS]] == expected_ids
|
| 80 |
+
page = result.pagination[SearchResourceType.USERS]
|
| 81 |
+
assert page.returned_count == len(expected_ids)
|
| 82 |
+
assert page.has_more is has_more
|
| 83 |
+
assert page.next_offset == (len(expected_ids) if has_more else None)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@pytest.mark.asyncio
|
| 87 |
+
async def test_candidates_preserves_successful_sections_when_one_provider_fails() -> None:
|
| 88 |
+
use_case = SearchCandidatesUseCase(
|
| 89 |
+
embedding_provider=StubEmbeddingProvider(),
|
| 90 |
+
providers=[
|
| 91 |
+
StubSearchProvider(SearchResourceType.USERS, [_hit("user", 0.9)]),
|
| 92 |
+
StubSearchProvider(
|
| 93 |
+
SearchResourceType.EVENTS,
|
| 94 |
+
error=TimeoutError("database timeout"),
|
| 95 |
+
),
|
| 96 |
+
],
|
| 97 |
+
policy_version="test-v1",
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
result = await use_case.execute(
|
| 101 |
+
query="consulta",
|
| 102 |
+
resource_types=(SearchResourceType.USERS, SearchResourceType.EVENTS),
|
| 103 |
+
candidate_limit_per_type=2,
|
| 104 |
+
as_of=datetime(2026, 7, 12, tzinfo=UTC),
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
assert [item.id for item in result.sections[SearchResourceType.USERS]] == ["user"]
|
| 108 |
+
assert result.sections[SearchResourceType.EVENTS] == []
|
| 109 |
+
assert result.failed_resources == {SearchResourceType.EVENTS: "TimeoutError"}
|
| 110 |
+
assert result.pagination[SearchResourceType.EVENTS].has_more is False
|
tests/test_search_relevance.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datetime import UTC, datetime
|
| 2 |
+
|
| 3 |
+
from app.modules.search.domain.filters import SearchCriteria, filter_and_rank_hits
|
| 4 |
+
from app.modules.search.domain.models import SearchHit, SearchResourceType
|
| 5 |
+
from app.modules.search.domain.relevance import SearchRelevanceThreshold
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _hit(
|
| 9 |
+
semantic_score: float | None,
|
| 10 |
+
lexical_score: float | None,
|
| 11 |
+
) -> SearchHit:
|
| 12 |
+
return SearchHit(
|
| 13 |
+
id="candidate",
|
| 14 |
+
resource_type=SearchResourceType.POSTS,
|
| 15 |
+
title="candidate",
|
| 16 |
+
score=0.9,
|
| 17 |
+
semantic_score=semantic_score,
|
| 18 |
+
lexical_score=lexical_score,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def test_relevance_threshold_accepts_either_absolute_signal() -> None:
|
| 23 |
+
threshold = SearchRelevanceThreshold(semantic_min=0.40, lexical_min=0.10)
|
| 24 |
+
|
| 25 |
+
assert threshold.accepts(_hit(0.40, None))
|
| 26 |
+
assert threshold.accepts(_hit(0.20, 0.10))
|
| 27 |
+
assert not threshold.accepts(_hit(0.399, 0.099))
|
| 28 |
+
assert not threshold.accepts(_hit(None, None))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def test_event_active_filter_keeps_in_progress_and_future_only() -> None:
|
| 32 |
+
active_at = datetime(2026, 7, 12, 12, 0, tzinfo=UTC)
|
| 33 |
+
criteria = SearchCriteria(event_active_at=active_at)
|
| 34 |
+
hits = [
|
| 35 |
+
SearchHit(
|
| 36 |
+
id="ended",
|
| 37 |
+
resource_type=SearchResourceType.EVENTS,
|
| 38 |
+
title="ended",
|
| 39 |
+
score=0.9,
|
| 40 |
+
semantic_score=0.9,
|
| 41 |
+
metadata={
|
| 42 |
+
"start_time": "2026-07-12T10:00:00Z",
|
| 43 |
+
"duration_minutes": 60,
|
| 44 |
+
},
|
| 45 |
+
),
|
| 46 |
+
SearchHit(
|
| 47 |
+
id="in-progress",
|
| 48 |
+
resource_type=SearchResourceType.EVENTS,
|
| 49 |
+
title="in-progress",
|
| 50 |
+
score=0.8,
|
| 51 |
+
semantic_score=0.8,
|
| 52 |
+
metadata={
|
| 53 |
+
"start_time": "2026-07-12T11:30:00Z",
|
| 54 |
+
"duration_minutes": 90,
|
| 55 |
+
},
|
| 56 |
+
),
|
| 57 |
+
SearchHit(
|
| 58 |
+
id="future",
|
| 59 |
+
resource_type=SearchResourceType.EVENTS,
|
| 60 |
+
title="future",
|
| 61 |
+
score=0.7,
|
| 62 |
+
semantic_score=0.7,
|
| 63 |
+
metadata={
|
| 64 |
+
"start_time": "2026-07-13T10:00:00Z",
|
| 65 |
+
"duration_minutes": 60,
|
| 66 |
+
},
|
| 67 |
+
),
|
| 68 |
+
SearchHit(
|
| 69 |
+
id="invalid",
|
| 70 |
+
resource_type=SearchResourceType.EVENTS,
|
| 71 |
+
title="invalid",
|
| 72 |
+
score=1.0,
|
| 73 |
+
semantic_score=1.0,
|
| 74 |
+
metadata={"start_time": "invalid", "duration_minutes": 60},
|
| 75 |
+
),
|
| 76 |
+
]
|
| 77 |
+
|
| 78 |
+
ranked = filter_and_rank_hits(SearchResourceType.EVENTS, hits, criteria)
|
| 79 |
+
|
| 80 |
+
assert [item.id for item in ranked] == ["in-progress", "future"]
|
tests/test_search_resource_documents.py
CHANGED
|
@@ -90,14 +90,50 @@ def test_event_identifier_tags_are_not_embedded_as_semantics() -> None:
|
|
| 90 |
"title": "Fiesta de Cumpleanos",
|
| 91 |
"description": "Evento totalmente independiente",
|
| 92 |
"is_public": True,
|
| 93 |
-
"tags": ["
|
|
|
|
| 94 |
}
|
| 95 |
)
|
| 96 |
assert record is not None
|
| 97 |
assert "b83ab97e" not in record.document
|
|
|
|
| 98 |
assert record.metadata["tag_ids"] == ["7", "b83ab97e-91a4-4f69-b102-b27c6092e9cb"]
|
| 99 |
|
| 100 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
def test_group_builds_private_search_acl_from_members_and_invites() -> None:
|
| 102 |
record = group_to_source_record(
|
| 103 |
{
|
|
|
|
| 90 |
"title": "Fiesta de Cumpleanos",
|
| 91 |
"description": "Evento totalmente independiente",
|
| 92 |
"is_public": True,
|
| 93 |
+
"tags": ["musica", "b83ab97e-91a4-4f69-b102-b27c6092e9cb"],
|
| 94 |
+
"tag_ids": ["7", "b83ab97e-91a4-4f69-b102-b27c6092e9cb"],
|
| 95 |
}
|
| 96 |
)
|
| 97 |
assert record is not None
|
| 98 |
assert "b83ab97e" not in record.document
|
| 99 |
+
assert "musica" in record.document
|
| 100 |
assert record.metadata["tag_ids"] == ["7", "b83ab97e-91a4-4f69-b102-b27c6092e9cb"]
|
| 101 |
|
| 102 |
|
| 103 |
+
def test_event_sync_normalizes_valid_temporal_metadata() -> None:
|
| 104 |
+
record = event_to_source_record(
|
| 105 |
+
{
|
| 106 |
+
"id": "event-valid",
|
| 107 |
+
"title": "Evento futuro",
|
| 108 |
+
"start_time": "2026-07-12T14:00:00-06:00",
|
| 109 |
+
"duration_minutes": "90",
|
| 110 |
+
}
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
assert record is not None
|
| 114 |
+
assert record.is_active is True
|
| 115 |
+
assert record.metadata["start_time"] == "2026-07-12T20:00:00+00:00"
|
| 116 |
+
assert record.metadata["duration_minutes"] == 90
|
| 117 |
+
assert record.metadata["temporal_metadata_valid"] is True
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def test_event_sync_deactivates_invalid_temporal_metadata() -> None:
|
| 121 |
+
record = event_to_source_record(
|
| 122 |
+
{
|
| 123 |
+
"id": "event-invalid",
|
| 124 |
+
"title": "Evento invalido",
|
| 125 |
+
"start_time": "not-a-date",
|
| 126 |
+
"duration_minutes": 60,
|
| 127 |
+
}
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
assert record is not None
|
| 131 |
+
assert record.is_active is False
|
| 132 |
+
assert record.metadata["temporal_metadata_valid"] is False
|
| 133 |
+
assert "start_time" not in record.metadata
|
| 134 |
+
assert "duration_minutes" not in record.metadata
|
| 135 |
+
|
| 136 |
+
|
| 137 |
def test_group_builds_private_search_acl_from_members_and_invites() -> None:
|
| 138 |
record = group_to_source_record(
|
| 139 |
{
|
tests/test_search_snapshot_sources.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pytest
|
| 2 |
+
|
| 3 |
+
from app.modules.clubs.infrastructure.main_api_club_source import MainApiClubsClient
|
| 4 |
+
from app.modules.events.infrastructure.main_api_event_source import MainApiEventsClient
|
| 5 |
+
from app.modules.groups.infrastructure.main_api_group_source import MainApiGroupsClient
|
| 6 |
+
from app.modules.users.infrastructure.main_api_user_source import MainApiUsersClient
|
| 7 |
+
from app.shared.config.settings import Settings
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def test_search_sources_use_internal_snapshot_paths() -> None:
|
| 11 |
+
settings = Settings(_env_file=None, ENV="local", MAIN_API_INTERNAL_TOKEN="internal")
|
| 12 |
+
|
| 13 |
+
assert MainApiUsersClient(settings)._path == "api/v1/internal/search/users/snapshot"
|
| 14 |
+
assert MainApiClubsClient(settings)._path == "api/v1/internal/search/clubs/snapshot"
|
| 15 |
+
assert MainApiGroupsClient(settings)._path == "api/v1/internal/search/groups/snapshot"
|
| 16 |
+
assert MainApiEventsClient(settings)._path == "api/v1/internal/search/events/snapshot"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_search_source_uses_only_internal_service_token() -> None:
|
| 20 |
+
settings = Settings(
|
| 21 |
+
_env_file=None,
|
| 22 |
+
ENV="local",
|
| 23 |
+
MAIN_API_INTERNAL_TOKEN="service-token",
|
| 24 |
+
MAIN_API_AUTH_TOKEN="user-jwt-must-not-be-used",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
assert MainApiUsersClient(settings)._build_headers() == {
|
| 28 |
+
"Authorization": "Bearer service-token"
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_search_source_rejects_missing_internal_token_even_with_user_jwt() -> None:
|
| 33 |
+
settings = Settings(
|
| 34 |
+
_env_file=None,
|
| 35 |
+
ENV="local",
|
| 36 |
+
MAIN_API_AUTH_TOKEN="user-jwt-must-not-be-used",
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
with pytest.raises(RuntimeError, match="MAIN_API_INTERNAL_TOKEN"):
|
| 40 |
+
MainApiUsersClient(settings)._build_headers()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_search_source_rejects_public_search_path() -> None:
|
| 44 |
+
settings = Settings(
|
| 45 |
+
_env_file=None,
|
| 46 |
+
ENV="local",
|
| 47 |
+
MAIN_API_INTERNAL_TOKEN="service-token",
|
| 48 |
+
MAIN_API_USERS_SNAPSHOT_PATH="/api/v1/users/search",
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
with pytest.raises(ValueError, match="internal snapshot"):
|
| 52 |
+
MainApiUsersClient(settings)
|
tests/test_search_threshold_calibration.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from scripts.calibrate_global_search_thresholds import LabeledScore, calibrate_resource
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def test_calibration_selects_thresholds_that_reject_negative_samples() -> None:
|
| 5 |
+
samples = [
|
| 6 |
+
LabeledScore("users", True, 0.80, None),
|
| 7 |
+
LabeledScore("users", True, 0.70, 0.20),
|
| 8 |
+
LabeledScore("users", False, 0.10, 0.01),
|
| 9 |
+
LabeledScore("users", False, 0.20, 0.02),
|
| 10 |
+
]
|
| 11 |
+
|
| 12 |
+
result = calibrate_resource(samples, target_precision=1.0)
|
| 13 |
+
|
| 14 |
+
assert result["precision"] == 1.0
|
| 15 |
+
assert result["recall"] == 1.0
|
| 16 |
+
assert result["false_positives"] == 0
|
tests/test_sql_contract.py
CHANGED
|
@@ -58,3 +58,48 @@ def test_post_feed_v2_and_convergent_schema_are_present() -> None:
|
|
| 58 |
assert "CREATE TABLE IF NOT EXISTS post_embeddings" in convergent
|
| 59 |
assert "CREATE OR REPLACE FUNCTION public.get_post_feed_features" in convergent
|
| 60 |
assert "migrate_post_feed_v2.sql" not in convergent # contenido inline, sin includes
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
assert "CREATE TABLE IF NOT EXISTS post_embeddings" in convergent
|
| 59 |
assert "CREATE OR REPLACE FUNCTION public.get_post_feed_features" in convergent
|
| 60 |
assert "migrate_post_feed_v2.sql" not in convergent # contenido inline, sin includes
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_global_search_candidate_filters_are_convergent_and_incremental() -> None:
|
| 64 |
+
contract = Path("sql/aws_pgvector_contract.sql").read_text(encoding="utf-8")
|
| 65 |
+
convergent = Path("sql/new_pgvector_schema.sql").read_text(encoding="utf-8")
|
| 66 |
+
migration = Path(
|
| 67 |
+
"sql/migrations/20260712_01_global_search_candidate_filters.sql"
|
| 68 |
+
).read_text(encoding="utf-8")
|
| 69 |
+
verifier = Path("sql/verify_global_search_candidate_filters.sql")
|
| 70 |
+
|
| 71 |
+
for token in (
|
| 72 |
+
"event_active_at",
|
| 73 |
+
"min_semantic_score",
|
| 74 |
+
"min_lexical_score",
|
| 75 |
+
"make_interval",
|
| 76 |
+
"try_parse_timestamptz",
|
| 77 |
+
"try_parse_positive_integer",
|
| 78 |
+
):
|
| 79 |
+
assert token in contract
|
| 80 |
+
assert token in convergent
|
| 81 |
+
assert token in migration
|
| 82 |
+
upper_migration = migration.upper()
|
| 83 |
+
assert "BEGIN;" in upper_migration
|
| 84 |
+
assert "COMMIT;" in upper_migration
|
| 85 |
+
assert "CREATE OR REPLACE FUNCTION PUBLIC.SEARCH_RESOURCE_EMBEDDINGS" in upper_migration
|
| 86 |
+
assert "CREATE TABLE" not in upper_migration
|
| 87 |
+
assert "ALTER TABLE" not in upper_migration
|
| 88 |
+
assert "CREATE INDEX" not in upper_migration
|
| 89 |
+
assert "DROP FUNCTION" not in upper_migration
|
| 90 |
+
assert "TO NLP_READER" in upper_migration
|
| 91 |
+
assert verifier.exists()
|
| 92 |
+
verifier_sql = verifier.read_text(encoding="utf-8")
|
| 93 |
+
assert "RAISE EXCEPTION 'Verify search V2" in verifier_sql
|
| 94 |
+
assert "SET TRANSACTION READ ONLY" in verifier_sql
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def test_global_search_sql_does_not_directly_cast_untrusted_event_metadata() -> None:
|
| 98 |
+
for path in (
|
| 99 |
+
Path("sql/aws_pgvector_contract.sql"),
|
| 100 |
+
Path("sql/new_pgvector_schema.sql"),
|
| 101 |
+
Path("sql/migrations/20260712_01_global_search_candidate_filters.sql"),
|
| 102 |
+
):
|
| 103 |
+
sql = path.read_text(encoding="utf-8")
|
| 104 |
+
assert "NULLIF(e.metadata->>'start_time', '')::timestamptz" not in sql
|
| 105 |
+
assert "(e.metadata->>'duration_minutes')::integer" not in sql
|