RAGBILTEN / app.py
senatus123's picture
fix: drop min_should in _in OR filter (bare should is required-OR)
9b0e830 verified
Raw History Blame Contribute Delete
69.3 kB
""" Combined HF Space app — Bilten + Odluke + Propisi RAG (search only, no reranking). Bilten: SentenceTransformer (BAAI/bge-m3) + Qdrant/bm25 sparse. Endpoints: /search. Odluke: Shared SentenceTransformer (BAAI/bge-m3) dense + Qdrant/bm25 sparse, RRF hybrid. Endpoints: /odluke/search_sparse, /odluke/search_hybrid, /odluke/search_set (preferred: one encode, dense + sparse + hybrid Qdrant, merged hits). Propisi: Shared SentenceTransformer (BAAI/bge-m3) dense + Qdrant/bm25 sparse, RRF hybrid. Endpoints: /propisi/search_dense, /propisi/search_sparse, /propisi/search_hybrid. Required env vars: Bilten: QDRANT_URL, QDRANT_API_KEY, COLLECTION_NAME (default: senatus-court-opinions-v4) Odluke: ODLUKE_QDRANT_URL, ODLUKE_QDRANT_API_KEY, ODLUKE_COLLECTION_NAME Propisi: PROPISI_QDRANT_URL, PROPISI_QDRANT_API_KEY, PROPISI_COLLECTION_NAME Shared: HF_TOKEN """ import os # HF / Docker images sometimes set OMP_NUM_THREADS to a value libgomp rejects (empty, "auto", etc.) # Align threads with actual core count: oversubscription (npr. OMP=8 na 2 vCPU) # uzrokuje thrashing i SPORIJI encode. Mora se postaviti PRIJE importa torch-a. _omp = os.environ.get("OMP_NUM_THREADS", "").strip() if not _omp or not _omp.isdigit() or int(_omp) < 1: os.environ["OMP_NUM_THREADS"] = str(os.cpu_count() or 4) import re import time import traceback import nh3 from fastapi import FastAPI, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel from typing import List, Optional, Dict, Any, Union from qdrant_client import QdrantClient, models from qdrant_client.models import FieldCondition, Filter, MatchText, MatchValue from sentence_transformers import SentenceTransformer from fastembed import SparseTextEmbedding import torch # Uskladi PyTorch CPU thread pool sa OMP-om (postavljen iznad prema cpu_count). try: torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "0")) or (os.cpu_count() or 4)) except Exception: pass def _onnx_runtime_has_cuda() -> bool: """HF Spaces often ship onnxruntime (CPU-only); CUDA EP requires onnxruntime-gpu matching CUDA.""" try: import onnxruntime as ort return "CUDAExecutionProvider" in ort.get_available_execution_providers() except Exception: return False # ============================================================================ # BILTEN CONFIG # ============================================================================ QDRANT_URL = os.environ.get("QDRANT_URL") QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY") HF_TOKEN = os.environ.get("HF_TOKEN") COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "senatus-court-opinions-v4") if not QDRANT_URL or not QDRANT_API_KEY: print("WARNING: QDRANT_URL or QDRANT_API_KEY not set. Bilten search will return 503.") # ============================================================================ # ODLUKE CONFIG (shared models; collection from env only) # ============================================================================ ODLUKE_QDRANT_URL = os.environ.get("ODLUKE_QDRANT_URL") or os.environ.get("QDRANT_URL") ODLUKE_QDRANT_API_KEY = os.environ.get("ODLUKE_QDRANT_API_KEY") or os.environ.get("QDRANT_API_KEY") ODLUKE_COLLECTION_NAME = (os.environ.get("ODLUKE_COLLECTION_NAME") or "").strip() ODLUKE_DENSE_VECTOR_NAME = os.environ.get("ODLUKE_DENSE_VECTOR_NAME", "dense") ODLUKE_SPARSE_VECTOR_NAME = os.environ.get("ODLUKE_SPARSE_VECTOR_NAME", "sparse") if not ODLUKE_QDRANT_URL or not ODLUKE_QDRANT_API_KEY: print("WARNING: ODLUKE_QDRANT_URL or ODLUKE_QDRANT_API_KEY not set. Odluke search will return 503.") if ODLUKE_QDRANT_URL and ODLUKE_QDRANT_API_KEY and not ODLUKE_COLLECTION_NAME: print("WARNING: ODLUKE_COLLECTION_NAME not set. Set it in Space secrets to enable Odluke search.") # ============================================================================ # PROPISI CONFIG (shared models; single collection) # ============================================================================ PROPISI_QDRANT_URL = os.environ.get("PROPISI_QDRANT_URL") or os.environ.get("QDRANT_URL") PROPISI_QDRANT_API_KEY = os.environ.get("PROPISI_QDRANT_API_KEY") or os.environ.get("QDRANT_API_KEY") PROPISI_COLLECTION_NAME = ( os.environ.get("PROPISI_COLLECTION_NAME") or os.environ.get("PROPISI_COLLECTION") or "senatus-propisi-v1" ).strip() PROPISI_DENSE_VECTOR_NAME = os.environ.get("PROPISI_DENSE_VECTOR_NAME", "dense") PROPISI_SPARSE_VECTOR_NAME = os.environ.get("PROPISI_SPARSE_VECTOR_NAME", "sparse") # Task 1 (synthetic-question search lane): separate pointer-only collection, # same Qdrant cluster/client, dense-only vector named "dense". Never mixed # into PROPISI_COLLECTION_NAME. PROPISI_QUESTIONS_COLLECTION_NAME = ( os.environ.get("PROPISI_QUESTIONS_COLLECTION_NAME") or "senatus-propisi-questions-v1" ).strip() # Search-time HNSW ef (recall lever). Kolekcija ima m=32/ef_construct=200, ali se # ef pretrage nigdje nije postavljao -> default. 160 podiže recall (da tačan član # ne ispadne iz kandidata) uz mali trošak latencije. Override preko env-a. try: PROPISI_HNSW_EF = int(os.environ.get("PROPISI_HNSW_EF", "160") or "160") if PROPISI_HNSW_EF < 1: PROPISI_HNSW_EF = 160 except Exception: PROPISI_HNSW_EF = 160 if not PROPISI_QDRANT_URL or not PROPISI_QDRANT_API_KEY: print("WARNING: PROPISI_QDRANT_URL or PROPISI_QDRANT_API_KEY not set. Propisi search will return 503.") # ============================================================================ # LOAD MODELS (shared between Bilten, Odluke and Propisi) # ============================================================================ device = "cuda" if torch.cuda.is_available() else "cpu" print(f"Using device: {device}") dense_model = None sparse_model = None needs_bilten = bool(QDRANT_URL and QDRANT_API_KEY) needs_odluke = bool(ODLUKE_QDRANT_URL and ODLUKE_QDRANT_API_KEY) needs_propisi = bool(PROPISI_QDRANT_URL and PROPISI_QDRANT_API_KEY) if needs_bilten or needs_odluke or needs_propisi: disable_onnx = os.environ.get("DISABLE_ONNX", "").strip().lower() in ("1", "true") use_onnx_cuda = ( device == "cuda" and not disable_onnx and _onnx_runtime_has_cuda() ) if device == "cuda" and not disable_onnx and not _onnx_runtime_has_cuda(): try: import onnxruntime as ort print( "ℹ️ BGE-M3: ONNX Runtime has no CUDAExecutionProvider " f"({ort.get_available_execution_providers()}) — typical on HF Spaces (CPU-only ORT build). " "Using PyTorch on GPU.", ) except Exception: print("ℹ️ BGE-M3: ONNX Runtime unavailable; using PyTorch on GPU.") if use_onnx_cuda: try: print("Loading shared dense model (BAAI/bge-m3 via SentenceTransformer ONNX+CUDA)...") dense_model = SentenceTransformer( "BAAI/bge-m3", device=device, token=HF_TOKEN, backend="onnx", model_kwargs={"provider": "CUDAExecutionProvider"}, ) print("✅ BGE-M3 loaded with ONNX backend (CUDAExecutionProvider)") except Exception as e: print(f"⚠️ ONNX load failed ({e}), falling back to PyTorch...") dense_model = SentenceTransformer("BAAI/bge-m3", device=device, token=HF_TOKEN) print("✅ BGE-M3 loaded with PyTorch backend (fallback)") else: print("Loading shared dense model (BAAI/bge-m3 via SentenceTransformer)...") dense_model = SentenceTransformer("BAAI/bge-m3", device=device, token=HF_TOKEN) print("✅ BGE-M3 loaded with PyTorch backend") print("Loading shared sparse model (Qdrant/bm25)...") sparse_model = SparseTextEmbedding("Qdrant/bm25") print("✅ Shared models loaded (dense + sparse)") # ============================================================================ # QDRANT CLIENTS # ============================================================================ bilten_qdrant_client = None if QDRANT_URL and QDRANT_API_KEY: bilten_qdrant_client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY, timeout=30) print(f"✅ Bilten Qdrant client initialized (collection: {COLLECTION_NAME})") odluke_qdrant_client = None if ODLUKE_QDRANT_URL and ODLUKE_QDRANT_API_KEY: odluke_qdrant_client = QdrantClient(url=ODLUKE_QDRANT_URL, api_key=ODLUKE_QDRANT_API_KEY, timeout=45) print(f"✅ Odluke Qdrant client initialized (collection: {ODLUKE_COLLECTION_NAME})") propisi_qdrant_client = None if PROPISI_QDRANT_URL and PROPISI_QDRANT_API_KEY: propisi_qdrant_client = QdrantClient(url=PROPISI_QDRANT_URL, api_key=PROPISI_QDRANT_API_KEY, timeout=60) print(f"✅ Propisi Qdrant client initialized (collection: {PROPISI_COLLECTION_NAME})") app = FastAPI(title="Senatus RAG (Bilten + Odluke + Propisi)") app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # ============================================================================ # PYDANTIC MODELS # ============================================================================ class SearchResult(BaseModel): id: str score: float content: str parent_id: str chunk_type: str metadata: Dict[str, Any] # ============================================================================ # HEALTH # ============================================================================ @app.get("/") def health(): bilten_ready = bilten_qdrant_client is not None and dense_model is not None odluke_ready = odluke_qdrant_client is not None and dense_model is not None and bool(ODLUKE_COLLECTION_NAME) propisi_ready = propisi_qdrant_client is not None and dense_model is not None and bool(PROPISI_COLLECTION_NAME) info = { "status": "ok", "device": device, "mode": "bilten_odluke_propisi", "models": { "dense": "BAAI/bge-m3 (SentenceTransformer, shared)", "sparse": "Qdrant/bm25 (fastembed, shared)", }, "features": {}, } if bilten_ready: info["features"]["bilten"] = { "collection": COLLECTION_NAME, "endpoints": ["/search"], } if odluke_ready: info["features"]["odluke"] = { "collection": ODLUKE_COLLECTION_NAME, "endpoints": [ "/odluke/search_sparse", "/odluke/search_hybrid", "/odluke/search_set", ], } if propisi_ready: info["features"]["propisi"] = { "collection": PROPISI_COLLECTION_NAME, "endpoints": ["/propisi/search_dense", "/propisi/search_sparse", "/propisi/search_hybrid"], } return info # ============================================================================ # BILTEN ENDPOINTS # ============================================================================ def _require_bilten(): if bilten_qdrant_client is None or dense_model is None: raise HTTPException(status_code=503, detail="Bilten search is not configured (QDRANT_URL, QDRANT_API_KEY)") # NOTE: sync `def` namjerno — FastAPI ga izvršava u threadpool-u, pa blokirajući # dense_model.encode(...) NE blokira event loop za ostale konkurentne korisnike. # (Ranije `async def` + blokirajući encode = serijalizacija svih zahtjeva.) @app.get("/search", response_model=List[SearchResult]) def search( q: str, top_k: int = 50, filter_case_number: Optional[str] = None, filter_case_reference_full: Optional[str] = None, filter_year: Optional[int] = None, filter_month: Optional[int] = None, filter_court: Optional[str] = None, filter_page: Optional[int] = None, tenant_id: Optional[str] = None ): _require_bilten() raw_query = q.strip() query = nh3.clean(raw_query.lower()) if not query: return [] filter_case_number_clean = nh3.clean(filter_case_number.strip()) if filter_case_number else None filter_case_ref_clean = nh3.clean(filter_case_reference_full.strip()) if filter_case_reference_full else None try: query_filter = None must_conditions = [] if tenant_id: must_conditions.append(models.FieldCondition(key="tenant_id", match=models.MatchValue(value=tenant_id))) case_conditions = [] if filter_case_number_clean: case_conditions.append(models.FieldCondition(key="case_number", match=models.MatchValue(value=filter_case_number_clean))) if filter_case_ref_clean: case_conditions.append(models.FieldCondition(key="case_reference_full", match=models.MatchValue(value=filter_case_ref_clean))) if case_conditions: if len(case_conditions) > 1: must_conditions.append(models.Filter(should=case_conditions)) else: must_conditions.append(case_conditions[0]) if filter_year: year_conditions = [ models.FieldCondition(key="case_year", match=models.MatchValue(value=filter_year)), models.FieldCondition(key="case_year", match=models.MatchValue(value=str(filter_year))), models.FieldCondition(key="godina_biltena", match=models.MatchValue(value=filter_year)), models.FieldCondition(key="godina_biltena", match=models.MatchValue(value=str(filter_year))) ] must_conditions.append(models.Filter(should=year_conditions)) if filter_month: must_conditions.append(models.FieldCondition(key="case_month", match=models.MatchValue(value=filter_month))) if filter_court: must_conditions.append(models.FieldCondition(key="court", match=models.MatchValue(value=filter_court))) if filter_page is not None and int(filter_page) > 0: p = int(filter_page) must_conditions.append( models.Filter( must=[ models.FieldCondition(key="page_start", range=models.Range(lte=p)), models.FieldCondition(key="page_end", range=models.Range(gte=p)), ] ) ) if must_conditions: query_filter = models.Filter(must=must_conditions) dense_query = dense_model.encode(query).tolist() sparse_result = list(sparse_model.query_embed(query))[0] sparse_query = models.SparseVector(indices=sparse_result.indices.tolist(), values=sparse_result.values.tolist()) results = None if filter_year: try: results = bilten_qdrant_client.query_points( collection_name=COLLECTION_NAME, prefetch=[ models.Prefetch(query=dense_query, using="dense", limit=top_k, filter=query_filter), models.Prefetch(query=sparse_query, using="bm25", limit=top_k, filter=query_filter), ], query=models.FusionQuery(fusion=models.Fusion.RRF), with_payload=True, limit=top_k, ) except Exception as filter_error: print(f"⚠️ Year filter error, retrying without year filter: {filter_error}") fallback_must = [] for cond in must_conditions: if isinstance(cond, models.Filter) and getattr(cond, 'should', None): is_year_filter = any( hasattr(fc, 'key') and fc.key in ['case_year', 'godina_biltena'] for fc in (cond.should or []) ) if not is_year_filter: fallback_must.append(cond) else: fallback_must.append(cond) fallback_filter = models.Filter(must=fallback_must) if fallback_must else None results = bilten_qdrant_client.query_points( collection_name=COLLECTION_NAME, prefetch=[ models.Prefetch(query=dense_query, using="dense", limit=top_k, filter=fallback_filter), models.Prefetch(query=sparse_query, using="bm25", limit=top_k, filter=fallback_filter), ], query=models.FusionQuery(fusion=models.Fusion.RRF), with_payload=True, limit=top_k, ) else: results = bilten_qdrant_client.query_points( collection_name=COLLECTION_NAME, prefetch=[ models.Prefetch(query=dense_query, using="dense", limit=top_k, filter=query_filter), models.Prefetch(query=sparse_query, using="bm25", limit=top_k, filter=query_filter), ], query=models.FusionQuery(fusion=models.Fusion.RRF), with_payload=True, limit=top_k, ) except Exception as e: print(f"Search error: {e}\n{traceback.format_exc()}") error_detail = f"Retrieval failed: {str(e)}" if len(str(e)) > 200: error_detail = f"Retrieval failed: {str(e)[:200]}..." raise HTTPException(status_code=500, detail=error_detail) final_results = [] for hit in results.points: payload = hit.payload or {} chunk_type = payload.get("chunk_type", "") if not chunk_type: content = payload.get("content", "").lower() chunk_type = "explanation" if ("obrazloženje" in content or len(content) > 500) else "opinion" final_results.append(SearchResult( id=str(hit.id), score=hit.score, content=payload.get("content", ""), parent_id=payload.get("parent_id", ""), chunk_type=chunk_type, metadata={k: v for k, v in payload.items() if k not in ["content"]} )) if filter_case_number_clean or filter_year: print(f"✅ Bilten filtered search: {len(final_results)} results") else: print(f"✅ Bilten hybrid search: {len(final_results)} results") return final_results # ============================================================================ # ODLUKE UTILITIES # ============================================================================ _DIACRITICS = { "ć": "c", "č": "c", "š": "s", "ž": "z", "đ": "dj", "Ć": "C", "Č": "C", "Š": "S", "Ž": "Z", "Đ": "Dj", } def strip_diacritics(text: str) -> str: for old, new in _DIACRITICS.items(): text = text.replace(old, new) return text.lower().strip() def _to_int(value, default=None): try: if value is None: return default return int(value) except (TypeError, ValueError): return default LEGAL_SYNONYMS: Dict[str, List[str]] = { "optuzeni": ["okrivljeni"], "okrivljeni": ["optuzeni"], "odbacena": ["odbijena", "odbijene", "odbijeni"], "odbijena": ["odbacena", "odbijene", "odbijeni", "odbijeno"], "odbijene": ["odbijena", "odbacena"], "oslobadjajuca": ["oslobodjen", "oslobodjen od optuzbe"], "oslobodjen": ["oslobadjajuca"], "osudjujuca": ["proglasen krivim", "oglasena krivom"], "proglasen": ["oglasena", "oglaseni"], "ukinuta": ["ukinuto", "ukinute"], "ukinuto": ["ukinuta"], "neizdrzavanje": ["nedavanje izdrzavanja"], "nedavanje": ["neizdrzavanje"], "drugostepena": ["po zalbi", "drugostepeno"], "prvostepena": ["prvostepeno", "prvostepeni"], "revizija": ["vanredni pravni lijek"], "usvojena": ["usvaja se"], "potvrdjuje": ["potvrdjena"], "potvrdjena": ["potvrdjuje", "potvrdjeno"], "zalba": ["zalbe", "zalbi", "zalbu"], "zalbe": ["zalba", "zalbi"], "preinacena": ["preinacuje", "preinaceno"], "preinacuje": ["preinacena"], "kradja": ["teska kradja", "kradja"], "ubistvo": ["tesko ubistvo"], } LAW_ABBREVIATIONS: Dict[str, str] = { "zpp": "zakon o parnicnom postupku", "zoo": "zakon o obligacionim odnosima", "zor": "zakon o radu", "kz": "krivicni zakonik", "kzcg": "krivicni zakonik crne gore", "zup": "zakon o upravnom postupku", "zs": "zakon o sudovima", "zio": "zakon o izvrsenju i obezbjedjenju", "zsp": "zakon o svojinsko pravnim odnosima", "zkp": "zakonik o krivicnom postupku", "zobs": "zakon o obligacionim odnosima i osnovama svojinsko pravnih odnosa", "zpd": "zakon o privrednim drustvima", "zus": "zakon o upravnom sporu", } _LAW_FULL_TO_ABBR: Dict[str, str] = {v: k for k, v in LAW_ABBREVIATIONS.items()} CLAN_INFLECTIONS = {"clan", "clana", "clanu", "clanom"} ZAKON_INFLECTIONS = {"zakon", "zakona", "zakonu", "zakonom"} ZAKONIK_INFLECTIONS = {"zakonik", "zakonika", "zakoniku", "zakonikom"} def _expand_word(word: str, qdrant_key: str) -> List: w = word.strip().lower() if not w: return [] alternatives = [] syns = LEGAL_SYNONYMS.get(w, []) if syns: alternatives.extend(syns) abbr_full = LAW_ABBREVIATIONS.get(w) if abbr_full: alternatives.append(abbr_full) for full_name, abbr in _LAW_FULL_TO_ABBR.items(): if w == full_name or (len(w) > 6 and w in full_name): alternatives.append(abbr) if w in CLAN_INFLECTIONS: alternatives.extend(f for f in CLAN_INFLECTIONS if f != w) if w in ZAKON_INFLECTIONS: alternatives.extend(f for f in ZAKON_INFLECTIONS if f != w) if w in ZAKONIK_INFLECTIONS: alternatives.extend(f for f in ZAKONIK_INFLECTIONS if f != w) if alternatives: should_conds = [FieldCondition(key=qdrant_key, match=MatchText(text=w))] for alt in alternatives: should_conds.append(FieldCondition(key=qdrant_key, match=MatchText(text=alt))) return [Filter(should=should_conds)] return [FieldCondition(key=qdrant_key, match=MatchText(text=w))] def _build_per_word_conditions(value: str, qdrant_key: str) -> List: raw_words = value.strip().lower().split() conditions = [] for w in raw_words: w_clean = re.sub(r"[^\w\u0107\u010d\u0161\u017e\u0111]", "", w) if w_clean: conditions.extend(_expand_word(w_clean, qdrant_key)) return conditions _ODLUKE_META_KEYS = ("case_number", "pravna_oblast", "predmet_spora", "ishod", "verdict_type_enriched", "krivicno_djelo", "pravni_osnov", "kazna", "sudije", "executive_summary", "naziv_suda", "naziv_odjeljenja", "vrsta_odluke", "vrsta_predmeta", "datum_vijecanja", "upisnik_godina", # Jina rerank contract (odluke_bgem3_sum normalized fields — must stay in sync with jinaRerankDocument.ts) "predmet_spora_normalized", "ishod_normalized", "sudije_normalized", "krivicno_djelo_normalized", "kazna_normalized") def _normalize_odluke_hit(payload: Dict[str, Any], score: float, point_id) -> Dict[str, Any]: meta = {k: payload.get(k) for k in _ODLUKE_META_KEYS} meta["vrsta_odjeljenja"] = payload.get("naziv_odjeljenja") content = payload.get("executive_summary") or "" if not content and payload.get("predmet_spora"): content = str(payload.get("predmet_spora", "")) + " " + str(payload.get("ishod", "")) content = (content or "").strip() return {"id": str(point_id), "score": float(score or 0.0), "content": content, "dbid": payload.get("dbid"), "page": 1, "metadata": meta} def _build_odluke_filter( filter_case_number: Optional[str] = None, filter_pravna_oblast: Optional[str] = None, filter_naziv_suda: Optional[str] = None, filter_naziv_suda_in: Optional[list[str]] = None, filter_exclude_naziv_suda: Optional[str] = None, filter_naziv_odjeljenja: Optional[str] = None, filter_naziv_odjeljenja_in: Optional[list[str]] = None, filter_exclude_naziv_odjeljenja: Optional[str] = None, filter_vrsta_odluke: Optional[str] = None, filter_vrsta_predmeta: Optional[str] = None, filter_ishod: Optional[str] = None, filter_year: Optional[int] = None, filter_krivicno_djelo: Optional[str] = None, filter_sudije: Optional[str] = None, filter_verdict_type: Optional[str] = None, filter_predmet_spora: Optional[str] = None, filter_pravni_osnov: Optional[str] = None, filter_kazna: Optional[str] = None, filter_datum_vijecanja: Optional[str] = None, filter_mode: Optional[str] = None, ) -> Optional[Filter]: """MUST / MUST_NOT only for Odluke search (no semantic SHOULD filters).""" must_conditions: list = [] must_not_conditions: list = [] if filter_naziv_suda_in: courts = [str(c).strip() for c in filter_naziv_suda_in if c and str(c).strip()] if courts: cond = _match_text_values_filter("naziv_suda", courts) if cond: must_conditions.append(cond) elif filter_naziv_suda: must_conditions.extend(_build_per_word_conditions(filter_naziv_suda, "naziv_suda")) if filter_exclude_naziv_suda: must_not_conditions.extend(_build_per_word_conditions(filter_exclude_naziv_suda, "naziv_suda")) if filter_year is not None and filter_year > 0: must_conditions.append(FieldCondition(key="upisnik_godina", match=MatchValue(value=filter_year))) if filter_case_number: must_conditions.append(FieldCondition(key="case_number_normalized", match=MatchText(text=filter_case_number))) if filter_sudije: must_conditions.append(FieldCondition(key="sudije_normalized", match=MatchText(text=filter_sudije))) if filter_datum_vijecanja: must_conditions.extend(_build_per_word_conditions(filter_datum_vijecanja, "datum_vijecanja")) if filter_naziv_odjeljenja_in: odjeljenja = [str(o).strip() for o in filter_naziv_odjeljenja_in if o and str(o).strip()] if odjeljenja: cond = _match_text_values_filter("naziv_odjeljenja", odjeljenja) if cond: must_conditions.append(cond) elif filter_naziv_odjeljenja: must_conditions.append(FieldCondition(key="naziv_odjeljenja", match=MatchText(text=filter_naziv_odjeljenja))) if filter_exclude_naziv_odjeljenja: must_not_conditions.extend(_build_per_word_conditions(filter_exclude_naziv_odjeljenja, "naziv_odjeljenja")) if not must_conditions and not must_not_conditions: return None parts: dict = {} if must_conditions: parts["must"] = must_conditions if must_not_conditions: parts["must_not"] = must_not_conditions return Filter(**parts) def _sanitize_filter(val: Optional[str], do_strip_diacritics: bool = True) -> Optional[str]: if not val: return None cleaned = nh3.clean(val.strip()) return strip_diacritics(cleaned) if do_strip_diacritics else cleaned def _log_odluke_scores(endpoint: str, output: list, top_n: int = 5): if not output: return print(f" [{endpoint}] SCORE BREAKDOWN ({len(output)} hits):") for i, hit in enumerate(output[:top_n]): cn = hit.get("metadata", {}).get("case_number", "?") dbid = hit.get("dbid", "?") print(f" #{i+1:>3} {cn:<22} score={hit['score']:.4f} dbid={dbid}") if len(output) > top_n: last = output[-1] cn = last.get("metadata", {}).get("case_number", "?") print(f" #{len(output):>3} {cn:<22} score={last['score']:.4f} (lowest)") # ============================================================================ # ODLUKE: common prep + internal search functions # ============================================================================ def _require_odluke(): if odluke_qdrant_client is None or dense_model is None: raise HTTPException(status_code=503, detail="Odluke search is not configured (ODLUKE_QDRANT_URL, ODLUKE_QDRANT_API_KEY)") if not ODLUKE_COLLECTION_NAME: raise HTTPException(status_code=503, detail="ODLUKE_COLLECTION_NAME not set. Set it in Space secrets.") def _prepare_odluke_query( q, collection, filter_case_number=None, filter_pravna_oblast=None, filter_naziv_suda=None, filter_naziv_suda_in=None, filter_exclude_naziv_suda=None, filter_naziv_odjeljenja=None, filter_naziv_odjeljenja_in=None, filter_exclude_naziv_odjeljenja=None, filter_vrsta_odluke=None, filter_vrsta_predmeta=None, filter_ishod=None, filter_year=None, filter_krivicno_djelo=None, filter_sudije=None, filter_verdict_type=None, filter_predmet_spora=None, filter_pravni_osnov=None, filter_kazna=None, filter_datum_vijecanja=None, filter_mode="must", ): """Sanitize inputs, build Qdrant filter. Returns (clean_q, coll, query_filter) or raises.""" coll = (collection or "").strip() or ODLUKE_COLLECTION_NAME clean_q = nh3.clean((q or "").strip().lower()) if not clean_q: return None, coll, None sf = { "case_number": _sanitize_filter(filter_case_number), "pravna_oblast": _sanitize_filter(filter_pravna_oblast), "naziv_suda": _sanitize_filter(filter_naziv_suda, do_strip_diacritics=False), "naziv_suda_in": [ c for c in ( _sanitize_filter(c, do_strip_diacritics=False) for c in (filter_naziv_suda_in or []) ) if c ] or None, "exclude_naziv_suda": _sanitize_filter(filter_exclude_naziv_suda, do_strip_diacritics=False), "naziv_odjeljenja": _sanitize_filter(filter_naziv_odjeljenja, do_strip_diacritics=False), "naziv_odjeljenja_in": [ o for o in ( _sanitize_filter(o, do_strip_diacritics=False) for o in (filter_naziv_odjeljenja_in or []) ) if o ] or None, "exclude_naziv_odjeljenja": _sanitize_filter(filter_exclude_naziv_odjeljenja, do_strip_diacritics=False), "vrsta_odluke": _sanitize_filter(filter_vrsta_odluke, do_strip_diacritics=False), "vrsta_predmeta": _sanitize_filter(filter_vrsta_predmeta, do_strip_diacritics=False), "ishod": _sanitize_filter(filter_ishod), "krivicno_djelo": _sanitize_filter(filter_krivicno_djelo), "sudije": _sanitize_filter(filter_sudije), "verdict_type": _sanitize_filter(filter_verdict_type), "predmet_spora": _sanitize_filter(filter_predmet_spora), "pravni_osnov": _sanitize_filter(filter_pravni_osnov), "kazna": _sanitize_filter(filter_kazna), "datum_vijecanja": _sanitize_filter(filter_datum_vijecanja, do_strip_diacritics=False), } query_filter = _build_odluke_filter( sf["case_number"], sf["pravna_oblast"], sf["naziv_suda"], sf["naziv_suda_in"], sf["exclude_naziv_suda"], sf["naziv_odjeljenja"], sf["naziv_odjeljenja_in"], sf["exclude_naziv_odjeljenja"], sf["vrsta_odluke"], sf["vrsta_predmeta"], sf["ishod"], _to_int(filter_year), sf["krivicno_djelo"], sf["sudije"], sf["verdict_type"], sf["predmet_spora"], sf["pravni_osnov"], sf["kazna"], sf["datum_vijecanja"], filter_mode or "must", ) return clean_q, coll, query_filter def _encode_odluke_dense(clean_q: str): return dense_model.encode(clean_q).tolist() def _encode_odluke_sparse(clean_q: str): sparse_result = list(sparse_model.query_embed(clean_q))[0] return models.SparseVector( indices=sparse_result.indices.tolist(), values=sparse_result.values.tolist(), ) def _odluke_dense_search_internal(clean_q: str, coll: str, top_k: int, query_filter, *, dense_vec=None): """Core dense vector search. Returns list of normalized hit dicts.""" t_enc = time.perf_counter() if dense_vec is None: dense_vec = _encode_odluke_dense(clean_q) encode_ms = (time.perf_counter() - t_enc) * 1000 t_search = time.perf_counter() results = odluke_qdrant_client.query_points( collection_name=coll, query=dense_vec, using=ODLUKE_DENSE_VECTOR_NAME, with_payload=True, limit=top_k, query_filter=query_filter, ) search_ms = (time.perf_counter() - t_search) * 1000 output = [_normalize_odluke_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] print(f"[{encode_ms + search_ms:.0f}ms] Odluke dense: {len(output)} results (encode={encode_ms:.0f}ms search={search_ms:.0f}ms)") return output def _odluke_sparse_search_internal(clean_q: str, coll: str, top_k: int, query_filter, *, sparse_query=None): """Core BM25 sparse search. Returns list of normalized hit dicts.""" t_enc = time.perf_counter() if sparse_query is None: sparse_query = _encode_odluke_sparse(clean_q) encode_ms = (time.perf_counter() - t_enc) * 1000 t_search = time.perf_counter() results = odluke_qdrant_client.query_points( collection_name=coll, query=sparse_query, using=ODLUKE_SPARSE_VECTOR_NAME, with_payload=True, limit=top_k, query_filter=query_filter, ) search_ms = (time.perf_counter() - t_search) * 1000 output = [_normalize_odluke_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] print(f"[{encode_ms + search_ms:.0f}ms] Odluke sparse: {len(output)} results (encode={encode_ms:.0f}ms search={search_ms:.0f}ms)") return output def _odluke_hybrid_search_internal( clean_q: str, coll: str, top_k: int, query_filter, *, dense_vec=None, sparse_query=None, ): """Core hybrid (dense + BM25, RRF fusion) search. Returns list of normalized hit dicts.""" t_enc = time.perf_counter() if dense_vec is None: dense_vec = _encode_odluke_dense(clean_q) if sparse_query is None: sparse_query = _encode_odluke_sparse(clean_q) encode_ms = (time.perf_counter() - t_enc) * 1000 t_search = time.perf_counter() results = odluke_qdrant_client.query_points( collection_name=coll, prefetch=[ models.Prefetch(query=dense_vec, using=ODLUKE_DENSE_VECTOR_NAME, limit=top_k, filter=query_filter), models.Prefetch(query=sparse_query, using=ODLUKE_SPARSE_VECTOR_NAME, limit=top_k, filter=query_filter), ], query=models.FusionQuery(fusion=models.Fusion.RRF), with_payload=True, limit=top_k, ) search_ms = (time.perf_counter() - t_search) * 1000 output = [_normalize_odluke_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] print(f"[{encode_ms + search_ms:.0f}ms] Odluke hybrid (RRF): {len(output)} results (encode={encode_ms:.0f}ms search={search_ms:.0f}ms)") return output # ============================================================================ # ODLUKE SEARCH ENDPOINTS # ============================================================================ @app.get("/odluke/search_sparse") def odluke_search_sparse( q: str = Query(..., min_length=1), top_k: int = Query(50, ge=1, le=200), collection: Optional[str] = None, filter_case_number: Optional[str] = None, filter_pravna_oblast: Optional[str] = None, filter_naziv_suda: Optional[str] = None, filter_exclude_naziv_suda: Optional[str] = None, filter_naziv_odjeljenja: Optional[str] = None, filter_exclude_naziv_odjeljenja: Optional[str] = None, filter_vrsta_odluke: Optional[str] = None, filter_vrsta_predmeta: Optional[str] = None, filter_ishod: Optional[str] = None, filter_year: Optional[int] = None, filter_krivicno_djelo: Optional[str] = None, filter_sudije: Optional[str] = None, filter_verdict_type: Optional[str] = None, filter_predmet_spora: Optional[str] = None, filter_pravni_osnov: Optional[str] = None, filter_kazna: Optional[str] = None, filter_datum_vijecanja: Optional[str] = None, filter_mode: Optional[str] = "must", ): _require_odluke() clean_q, coll, query_filter = _prepare_odluke_query( q, collection, filter_case_number, filter_pravna_oblast, filter_naziv_suda, None, filter_exclude_naziv_suda, filter_naziv_odjeljenja, None, filter_exclude_naziv_odjeljenja, filter_vrsta_odluke, filter_vrsta_predmeta, filter_ishod, filter_year, filter_krivicno_djelo, filter_sudije, filter_verdict_type, filter_predmet_spora, filter_pravni_osnov, filter_kazna, filter_datum_vijecanja, filter_mode, ) if not clean_q: return {"hits": [], "collection": coll} try: output = _odluke_sparse_search_internal(clean_q, coll, top_k, query_filter) _log_odluke_scores("sparse", output) return {"hits": output, "collection": coll} except HTTPException: raise except Exception as e: print(f"❌ Odluke sparse search error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Odluke sparse search failed: {str(e)[:300]}") @app.get("/odluke/search_hybrid") def odluke_search_hybrid( q: str = Query(..., min_length=1), top_k: int = Query(50, ge=1, le=200), collection: Optional[str] = None, filter_case_number: Optional[str] = None, filter_pravna_oblast: Optional[str] = None, filter_naziv_suda: Optional[str] = None, filter_exclude_naziv_suda: Optional[str] = None, filter_naziv_odjeljenja: Optional[str] = None, filter_exclude_naziv_odjeljenja: Optional[str] = None, filter_vrsta_odluke: Optional[str] = None, filter_vrsta_predmeta: Optional[str] = None, filter_ishod: Optional[str] = None, filter_year: Optional[int] = None, filter_krivicno_djelo: Optional[str] = None, filter_sudije: Optional[str] = None, filter_verdict_type: Optional[str] = None, filter_predmet_spora: Optional[str] = None, filter_pravni_osnov: Optional[str] = None, filter_kazna: Optional[str] = None, filter_datum_vijecanja: Optional[str] = None, ): """Hybrid search: dense+sparse (BM25) fused via Qdrant RRF. MUST / MUST_NOT filters only.""" _require_odluke() clean_q, coll, query_filter = _prepare_odluke_query( q, collection, filter_case_number, filter_pravna_oblast, filter_naziv_suda, None, filter_exclude_naziv_suda, filter_naziv_odjeljenja, None, filter_exclude_naziv_odjeljenja, filter_vrsta_odluke, filter_vrsta_predmeta, filter_ishod, filter_year, filter_krivicno_djelo, filter_sudije, filter_verdict_type, filter_predmet_spora, filter_pravni_osnov, filter_kazna, filter_datum_vijecanja, "must", ) if not clean_q: return {"hits": [], "collection": coll} try: output = _odluke_hybrid_search_internal(clean_q, coll, top_k, query_filter) _log_odluke_scores("hybrid", output) return {"hits": output, "collection": coll} except HTTPException: raise except Exception as e: print(f"❌ Odluke hybrid search error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Odluke hybrid search failed: {str(e)[:300]}") class OdlukeSearchRequest(BaseModel): q: str top_k: int = 100 collection: Optional[str] = None filter_case_number: Optional[str] = None filter_pravna_oblast: Optional[str] = None filter_naziv_suda: Optional[str] = None filter_naziv_suda_in: Optional[List[str]] = None filter_exclude_naziv_suda: Optional[str] = None filter_naziv_odjeljenja: Optional[str] = None filter_naziv_odjeljenja_in: Optional[List[str]] = None filter_exclude_naziv_odjeljenja: Optional[str] = None filter_vrsta_odluke: Optional[str] = None filter_vrsta_predmeta: Optional[str] = None filter_ishod: Optional[str] = None filter_year: Optional[int] = None filter_krivicno_djelo: Optional[str] = None filter_sudije: Optional[str] = None filter_verdict_type: Optional[str] = None filter_predmet_spora: Optional[str] = None filter_pravni_osnov: Optional[str] = None filter_kazna: Optional[str] = None filter_datum_vijecanja: Optional[str] = None filter_mode: Optional[str] = "must" class OdlukeSearchSetRequest(OdlukeSearchRequest): dense_top_k: Optional[int] = None sparse_top_k: Optional[int] = None hybrid_top_k: Optional[int] = None q_dense: Optional[str] = None q_sparse: Optional[str] = None def _with_odluke_lane_meta(hits: list[dict[str, Any]], lane: str) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] for rank, hit in enumerate(hits, 1): item = dict(hit) meta = dict(item.get("metadata") or {}) meta[f"{lane}_rank"] = rank meta[f"{lane}_score"] = item.get("score") lanes = list(meta.get("search_lanes") or []) if lane not in lanes: lanes.append(lane) meta["search_lanes"] = lanes item["metadata"] = meta out.append(item) return out def _merge_odluke_search_set( dense_hits: list[dict[str, Any]], sparse_hits: list[dict[str, Any]], hybrid_hits: list[dict[str, Any]], limit: int, ) -> list[dict[str, Any]]: """One presuda = one chunk; merge lanes by dbid for combined hits list.""" by_key: dict[str, dict[str, Any]] = {} for lane, hits in ( ("dense", dense_hits), ("sparse", sparse_hits), ("hybrid", hybrid_hits), ): for rank, hit in enumerate(hits, 1): dbid = str(hit.get("dbid") or hit.get("id") or "") key = dbid or f"{lane}:{rank}" lane_signal = 1.0 / (rank + 20) existing = by_key.get(key) if existing is None: merged = dict(hit) merged_meta = dict(merged.get("metadata") or {}) merged_meta["search_lanes"] = [lane] merged_meta[f"{lane}_rank"] = rank merged_meta[f"{lane}_score"] = hit.get("score") merged["_searchset_score"] = lane_signal merged["metadata"] = merged_meta by_key[key] = merged continue existing_meta = dict(existing.get("metadata") or {}) lanes = list(existing_meta.get("search_lanes") or []) if lane not in lanes: lanes.append(lane) existing_meta["search_lanes"] = lanes existing_meta[f"{lane}_rank"] = rank existing_meta[f"{lane}_score"] = hit.get("score") existing["metadata"] = existing_meta existing["_searchset_score"] = float(existing.get("_searchset_score") or 0) + lane_signal if float(hit.get("score") or 0) > float(existing.get("score") or 0): existing["score"] = hit.get("score") merged = sorted( by_key.values(), key=lambda h: ( float(h.get("_searchset_score") or 0), float(h.get("score") or 0), ), reverse=True, ) for item in merged: item.pop("_searchset_score", None) return merged[:limit] @app.post("/odluke/search_set") def odluke_search_set_post(body: OdlukeSearchSetRequest): _require_odluke() top_k = _clamp_top_k(body.top_k) dense_top_k = _clamp_top_k(body.dense_top_k or top_k) sparse_top_k = _clamp_top_k(body.sparse_top_k or top_k) hybrid_top_k = _clamp_top_k(body.hybrid_top_k or top_k) clean_q, coll, query_filter = _prepare_odluke_query( body.q, body.collection, body.filter_case_number, body.filter_pravna_oblast, body.filter_naziv_suda, body.filter_naziv_suda_in, body.filter_exclude_naziv_suda, body.filter_naziv_odjeljenja, body.filter_naziv_odjeljenja_in, body.filter_exclude_naziv_odjeljenja, body.filter_vrsta_odluke, body.filter_vrsta_predmeta, body.filter_ishod, body.filter_year, body.filter_krivicno_djelo, body.filter_sudije, body.filter_verdict_type, body.filter_predmet_spora, body.filter_pravni_osnov, body.filter_kazna, body.filter_datum_vijecanja, body.filter_mode, ) if not clean_q: return {"hits": [], "dense": [], "sparse": [], "hybrid": [], "collection": coll} try: dense_q = nh3.clean(str(body.q_dense or body.q or "").strip()) or clean_q sparse_q = nh3.clean(str(body.q_sparse or body.q or "").strip()) or clean_q t_enc = time.perf_counter() dense_vec = _encode_odluke_dense(dense_q) sparse_query = _encode_odluke_sparse(sparse_q) encode_ms = (time.perf_counter() - t_enc) * 1000 print(f"[{encode_ms:.0f}ms] Odluke search_set encode (dense+sparse, shared for all lanes)") t_dense = time.perf_counter() dense = _with_odluke_lane_meta( _odluke_dense_search_internal( clean_q, coll, dense_top_k, query_filter, dense_vec=dense_vec, ), "dense", ) dense_lane_ms = (time.perf_counter() - t_dense) * 1000 t_sparse = time.perf_counter() sparse = _with_odluke_lane_meta( _odluke_sparse_search_internal( clean_q, coll, sparse_top_k, query_filter, sparse_query=sparse_query, ), "sparse", ) sparse_lane_ms = (time.perf_counter() - t_sparse) * 1000 t_hybrid = time.perf_counter() hybrid = _with_odluke_lane_meta( _odluke_hybrid_search_internal( clean_q, coll, hybrid_top_k, query_filter, dense_vec=dense_vec, sparse_query=sparse_query, ), "hybrid", ) hybrid_lane_ms = (time.perf_counter() - t_hybrid) * 1000 hits = _merge_odluke_search_set(dense, sparse, hybrid, top_k) print( f"Odluke search_set lanes: dense={len(dense)} sparse={len(sparse)} hybrid={len(hybrid)} " f"merged={len(hits)} (encode={encode_ms:.0f}ms dense={dense_lane_ms:.0f}ms " f"sparse={sparse_lane_ms:.0f}ms hybrid={hybrid_lane_ms:.0f}ms)" ) return { "hits": hits, "dense": dense, "sparse": sparse, "hybrid": hybrid, "collection": coll, "meta": { "dense_top_k": dense_top_k, "sparse_top_k": sparse_top_k, "hybrid_top_k": hybrid_top_k, "combined": len(hits), "timings": { "encode_ms": round(encode_ms, 1), "dense_lane_ms": round(dense_lane_ms, 1), "sparse_lane_ms": round(sparse_lane_ms, 1), "hybrid_lane_ms": round(hybrid_lane_ms, 1), "total_ms": round(encode_ms + dense_lane_ms + sparse_lane_ms + hybrid_lane_ms, 1), }, }, } except HTTPException: raise except Exception as e: print(f"❌ Odluke search_set error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Odluke search_set failed: {str(e)[:300]}") # ============================================================================ # PROPISI UTILITIES # ============================================================================ _PROPISI_META_KEYS = ( "dbid", "chunk_id", "chunk_type", "corpus_scope", "vrsta_akta", "status", "datum_objavljivanja", "datum_stupanja_na_snagu", "clan_int", "clan_suffix", "stav_broj", "tacka_broj", "parent_chunk_id", "parent_law_dbid", "parent_law_naziv", "naziv", "izdanje", "donosilac", "oblast", "pozicija", "marginal_title", "page_first", "page_last", "is_amendment", "has_amendment_targets", "amendment_target_keys", "amendment_target_clans", "amendment_target_clan_ints", "amendment_target_stavs", "amendment_actions", "amendment_action_buckets", "amendment_parent_candidate_dbids", "amendment_parent_confidence", "amendment_parent_confidences", "amends_or_amended_by_dbids", "podzakonski_dbids", "donesen_na_osnovu_dbids", "inferred_basis_keys", "inferred_donesen_na_osnovu_dbids", "predecessor_dbids", "truncated_podzakonski", ) def _require_propisi(): if propisi_qdrant_client is None or dense_model is None: raise HTTPException(status_code=503, detail="Propisi search is not configured (PROPISI_QDRANT_URL, PROPISI_QDRANT_API_KEY)") if not PROPISI_COLLECTION_NAME: raise HTTPException(status_code=503, detail="PROPISI_COLLECTION_NAME not set.") class PropisiSearchRequest(BaseModel): q: str top_k: int = 80 collection: Optional[str] = None filter_vrsta_akta: Optional[Union[str, List[str]]] = None filter_corpus_scope: Optional[Union[str, List[str]]] = None filter_dbid_in: Optional[Union[str, List[str]]] = None filter_status: Optional[Union[str, List[str]]] = "Vazeci" filter_chunk_type: Optional[Union[str, List[str]]] = None filter_year_from: Optional[int] = None class PropisiQuestionLaneRequest(BaseModel): q: str top_k: int = 20 class PropisiSearchSetRequest(PropisiSearchRequest): dense_top_k: Optional[int] = None sparse_top_k: Optional[int] = None hybrid_top_k: Optional[int] = None q_dense: Optional[str] = None q_sparse: Optional[str] = None def _csv_values(value: Any) -> list[str]: if not value: return [] if isinstance(value, list): raw_parts = value else: raw_parts = str(value).split(",") out: list[str] = [] for raw in raw_parts: cleaned = nh3.clean(str(raw).strip()) if not cleaned or cleaned.lower() in {"all", "any", "null", "none"}: continue out.extend(part.strip() for part in cleaned.split(",") if part.strip()) return out def _status_values(value: Any) -> list[str]: values = _csv_values(value) out: list[str] = [] for v in values: out.append(v) if v == "Važeći": out.append("Vazeci") elif v == "Vazeci": out.append("Važeći") deduped: list[str] = [] for v in out: if v not in deduped: deduped.append(v) return deduped def _match_values_condition(key: str, values: list[str]): if not values: return None if len(values) == 1: return FieldCondition(key=key, match=MatchValue(value=values[0])) return FieldCondition(key=key, match=models.MatchAny(any=values)) def _match_text_values_filter(key: str, values: list[str]): """OR filter for text-indexed fields (naziv_suda, naziv_odjeljenja are text, not keyword). MatchAny/MatchValue return 0 on text index, so build a nested Filter with only `should` (one MatchText per value). A Filter with only `should` (no must) acts as OR in Qdrant. NOTE: do NOT add min_should — empirically returns 0 with MatchText; bare should is required-OR. Returns a nested Filter (must = AND of this OR group).""" vals = [str(v).strip() for v in values if v and str(v).strip()] if not vals: return None should = [FieldCondition(key=key, match=MatchText(text=v)) for v in vals] return Filter(should=should) def _normalize_propisi_hit(payload: Dict[str, Any], score: float, point_id) -> Dict[str, Any]: meta = {k: payload.get(k) for k in _PROPISI_META_KEYS if k in payload} content = str(payload.get("content") or "").strip() return { "id": str(point_id), "score": float(score or 0.0), "content": content, "dbid": payload.get("dbid"), "parent_id": payload.get("dbid") or "", "chunk_type": payload.get("chunk_type") or "", "metadata": meta, } def _normalize_propisi_question_hit(payload: Dict[str, Any], score: float, point_id) -> Dict[str, Any]: """Task 1c: pointer-only hit from senatus-propisi-questions-v1. Deliberately minimal — dbid/clan_int/question_text/naziv only, matching the collection's own minimal schema (see create_question_collection.py). The caller (rag-propisi-v2) must resolve this back to a live article_full hit in the main collection before it can be used for anything beyond ranking; question_text must never reach evidence/citations. """ return { "id": str(point_id), "score": float(score or 0.0), "dbid": payload.get("dbid"), "clan_int": payload.get("clan_int"), "question_text": payload.get("question_text") or "", "naziv": payload.get("naziv") or "", } def _propisi_question_lane_search_internal(clean_q: str, top_k: int): dense_vec = _encode_propisi_dense(clean_q) results = propisi_qdrant_client.query_points( collection_name=PROPISI_QUESTIONS_COLLECTION_NAME, query=dense_vec, using=PROPISI_DENSE_VECTOR_NAME, with_payload=True, limit=top_k, search_params=models.SearchParams(hnsw_ef=PROPISI_HNSW_EF), ) return [_normalize_propisi_question_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] def _build_propisi_filter( filter_vrsta_akta: Any = None, filter_corpus_scope: Any = None, filter_dbid_in: Any = None, filter_status: Any = "Vazeci", filter_chunk_type: Any = None, filter_year_from: Optional[int] = None, ) -> Optional[Filter]: must_conditions = [] for key, raw_values in ( ("vrsta_akta", _csv_values(filter_vrsta_akta)), ("corpus_scope", _csv_values(filter_corpus_scope)), ("dbid", _csv_values(filter_dbid_in)), ("status", _status_values(filter_status)), ("chunk_type", _csv_values(filter_chunk_type)), ): cond = _match_values_condition(key, raw_values) if cond is not None: must_conditions.append(cond) if filter_year_from is not None and int(filter_year_from) > 0: datetime_range = getattr(models, "DatetimeRange", None) if datetime_range is None: raise HTTPException(status_code=500, detail="qdrant-client lacks DatetimeRange support for filter_year_from") must_conditions.append( FieldCondition( key="datum_objavljivanja", range=datetime_range(gte=f"{int(filter_year_from):04d}-01-01T00:00:00Z"), ) ) if not must_conditions: return None return Filter(must=must_conditions) def _prepare_propisi_query( q: str, collection: Optional[str], filter_vrsta_akta: Any = None, filter_corpus_scope: Any = None, filter_dbid_in: Any = None, filter_status: Any = "Vazeci", filter_chunk_type: Any = None, filter_year_from: Optional[int] = None, ): coll = (collection or "").strip() or PROPISI_COLLECTION_NAME clean_q = nh3.clean((q or "").strip().lower()) if not clean_q: return None, coll, None query_filter = _build_propisi_filter( filter_vrsta_akta=filter_vrsta_akta, filter_corpus_scope=filter_corpus_scope, filter_dbid_in=filter_dbid_in, filter_status=filter_status, filter_chunk_type=filter_chunk_type, filter_year_from=filter_year_from, ) return clean_q, coll, query_filter def _clamp_top_k(top_k: int) -> int: try: parsed = int(top_k) except Exception: parsed = 80 return max(1, min(parsed, 200)) def _encode_propisi_dense(clean_q: str): return dense_model.encode(clean_q).tolist() def _encode_propisi_sparse(clean_q: str): sparse_result = list(sparse_model.query_embed(clean_q))[0] return models.SparseVector( indices=sparse_result.indices.tolist(), values=sparse_result.values.tolist(), ) def _propisi_dense_search_internal( clean_q: str, coll: str, top_k: int, query_filter, *, dense_vec=None, ): t_enc = time.perf_counter() if dense_vec is None: dense_vec = _encode_propisi_dense(clean_q) encode_ms = (time.perf_counter() - t_enc) * 1000 t_search = time.perf_counter() results = propisi_qdrant_client.query_points( collection_name=coll, query=dense_vec, using=PROPISI_DENSE_VECTOR_NAME, with_payload=True, limit=top_k, query_filter=query_filter, search_params=models.SearchParams(hnsw_ef=PROPISI_HNSW_EF), ) search_ms = (time.perf_counter() - t_search) * 1000 output = [_normalize_propisi_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] print(f"[{encode_ms + search_ms:.0f}ms] Propisi dense: {len(output)} results") return output def _propisi_sparse_search_internal( clean_q: str, coll: str, top_k: int, query_filter, *, sparse_query=None, ): t_enc = time.perf_counter() if sparse_query is None: sparse_query = _encode_propisi_sparse(clean_q) encode_ms = (time.perf_counter() - t_enc) * 1000 t_search = time.perf_counter() results = propisi_qdrant_client.query_points( collection_name=coll, query=sparse_query, using=PROPISI_SPARSE_VECTOR_NAME, with_payload=True, limit=top_k, query_filter=query_filter, ) search_ms = (time.perf_counter() - t_search) * 1000 output = [_normalize_propisi_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] print(f"[{encode_ms + search_ms:.0f}ms] Propisi sparse: {len(output)} results") return output def _propisi_hybrid_search_internal( clean_q: str, coll: str, top_k: int, query_filter, *, dense_vec=None, sparse_query=None, ): t_enc = time.perf_counter() if dense_vec is None: dense_vec = _encode_propisi_dense(clean_q) if sparse_query is None: sparse_query = _encode_propisi_sparse(clean_q) encode_ms = (time.perf_counter() - t_enc) * 1000 t_search = time.perf_counter() results = propisi_qdrant_client.query_points( collection_name=coll, prefetch=[ models.Prefetch( query=dense_vec, using=PROPISI_DENSE_VECTOR_NAME, limit=top_k, filter=query_filter, params=models.SearchParams(hnsw_ef=PROPISI_HNSW_EF), ), models.Prefetch(query=sparse_query, using=PROPISI_SPARSE_VECTOR_NAME, limit=top_k, filter=query_filter), ], query=models.FusionQuery(fusion=models.Fusion.RRF), with_payload=True, limit=top_k, ) search_ms = (time.perf_counter() - t_search) * 1000 output = [_normalize_propisi_hit(p.payload or {}, p.score or 0.0, p.id) for p in results.points] print(f"[{encode_ms + search_ms:.0f}ms] Propisi hybrid (RRF): {len(output)} results") return output def _with_propisi_lane_meta(hits: list[dict[str, Any]], lane: str) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] for rank, hit in enumerate(hits, 1): item = dict(hit) meta = dict(item.get("metadata") or {}) meta[f"{lane}_rank"] = rank meta[f"{lane}_score"] = item.get("score") lanes = list(meta.get("search_lanes") or []) if lane not in lanes: lanes.append(lane) meta["search_lanes"] = lanes item["metadata"] = meta out.append(item) return out def _merge_propisi_search_set( dense_hits: list[dict[str, Any]], sparse_hits: list[dict[str, Any]], hybrid_hits: list[dict[str, Any]], limit: int, ) -> list[dict[str, Any]]: by_key: dict[str, dict[str, Any]] = {} for lane, hits in ( ("dense", dense_hits), ("sparse", sparse_hits), ("hybrid", hybrid_hits), ): for rank, hit in enumerate(hits, 1): meta = dict(hit.get("metadata") or {}) key = str(meta.get("chunk_id") or hit.get("id") or f"{hit.get('dbid')}:{rank}:{lane}") existing = by_key.get(key) lane_signal = 1.0 / (rank + 20) if existing is None: merged = dict(hit) merged_meta = dict(meta) merged_meta["search_lanes"] = [lane] merged_meta[f"{lane}_rank"] = rank merged_meta[f"{lane}_score"] = hit.get("score") merged["_searchset_score"] = lane_signal merged["metadata"] = merged_meta by_key[key] = merged continue existing_meta = dict(existing.get("metadata") or {}) lanes = list(existing_meta.get("search_lanes") or []) if lane not in lanes: lanes.append(lane) existing_meta["search_lanes"] = lanes existing_meta[f"{lane}_rank"] = rank existing_meta[f"{lane}_score"] = hit.get("score") existing["metadata"] = existing_meta existing["_searchset_score"] = float(existing.get("_searchset_score") or 0) + lane_signal if float(hit.get("score") or 0) > float(existing.get("score") or 0): existing["score"] = hit.get("score") merged = sorted( by_key.values(), key=lambda h: ( float(h.get("_searchset_score") or 0), float(h.get("score") or 0), ), reverse=True, ) for item in merged: item.pop("_searchset_score", None) return merged[:limit] # ============================================================================ # PROPISI SEARCH ENDPOINTS # ============================================================================ @app.get("/propisi/search_dense") def propisi_search_dense( q: str = Query(..., min_length=1), top_k: int = Query(80, ge=1, le=200), collection: Optional[str] = None, filter_vrsta_akta: Optional[str] = None, filter_corpus_scope: Optional[str] = None, filter_dbid_in: Optional[str] = None, filter_status: Optional[str] = "Vazeci", filter_chunk_type: Optional[str] = None, filter_year_from: Optional[int] = None, ): _require_propisi() clean_q, coll, query_filter = _prepare_propisi_query( q, collection, filter_vrsta_akta, filter_corpus_scope, filter_dbid_in, filter_status, filter_chunk_type, filter_year_from, ) if not clean_q: return {"hits": [], "collection": coll} try: return {"hits": _propisi_dense_search_internal(clean_q, coll, top_k, query_filter), "collection": coll} except HTTPException: raise except Exception as e: print(f"❌ Propisi dense search error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Propisi dense search failed: {str(e)[:300]}") @app.get("/propisi/search_sparse") def propisi_search_sparse( q: str = Query(..., min_length=1), top_k: int = Query(80, ge=1, le=200), collection: Optional[str] = None, filter_vrsta_akta: Optional[str] = None, filter_corpus_scope: Optional[str] = None, filter_dbid_in: Optional[str] = None, filter_status: Optional[str] = "Vazeci", filter_chunk_type: Optional[str] = None, filter_year_from: Optional[int] = None, ): _require_propisi() clean_q, coll, query_filter = _prepare_propisi_query( q, collection, filter_vrsta_akta, filter_corpus_scope, filter_dbid_in, filter_status, filter_chunk_type, filter_year_from, ) if not clean_q: return {"hits": [], "collection": coll} try: return {"hits": _propisi_sparse_search_internal(clean_q, coll, top_k, query_filter), "collection": coll} except HTTPException: raise except Exception as e: print(f"❌ Propisi sparse search error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Propisi sparse search failed: {str(e)[:300]}") @app.get("/propisi/search_hybrid") def propisi_search_hybrid( q: str = Query(..., min_length=1), top_k: int = Query(80, ge=1, le=200), collection: Optional[str] = None, filter_vrsta_akta: Optional[str] = None, filter_corpus_scope: Optional[str] = None, filter_dbid_in: Optional[str] = None, filter_status: Optional[str] = "Vazeci", filter_chunk_type: Optional[str] = None, filter_year_from: Optional[int] = None, ): return _propisi_search_hybrid_response( q=q, top_k=top_k, collection=collection, filter_vrsta_akta=filter_vrsta_akta, filter_corpus_scope=filter_corpus_scope, filter_dbid_in=filter_dbid_in, filter_status=filter_status, filter_chunk_type=filter_chunk_type, filter_year_from=filter_year_from, ) @app.post("/propisi/search_hybrid") def propisi_search_hybrid_post(body: PropisiSearchRequest): return _propisi_search_hybrid_response( q=body.q, top_k=body.top_k, collection=body.collection, filter_vrsta_akta=body.filter_vrsta_akta, filter_corpus_scope=body.filter_corpus_scope, filter_dbid_in=body.filter_dbid_in, filter_status=body.filter_status, filter_chunk_type=body.filter_chunk_type, filter_year_from=body.filter_year_from, ) @app.post("/propisi/search_questions") def propisi_search_questions_post(body: PropisiQuestionLaneRequest): """Task 1c: synthetic-question search lane (senatus-propisi-questions-v1). Dense-only, no filters (nothing to filter on in this pointer-only collection — see create_question_collection.py). Returns pointer hits only (dbid, clan_int, question_text, naziv); the caller must resolve each hit to a live article_full chunk in senatus-propisi-v1 before use. """ _require_propisi() if not PROPISI_QUESTIONS_COLLECTION_NAME: raise HTTPException(status_code=503, detail="PROPISI_QUESTIONS_COLLECTION_NAME not set.") top_k = _clamp_top_k(body.top_k) clean_q = nh3.clean((body.q or "").strip().lower()) if not clean_q: return {"hits": [], "collection": PROPISI_QUESTIONS_COLLECTION_NAME} try: hits = _propisi_question_lane_search_internal(clean_q, top_k) return {"hits": hits, "collection": PROPISI_QUESTIONS_COLLECTION_NAME} except HTTPException: raise except Exception as e: print(f"❌ Propisi search_questions error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Propisi search_questions failed: {str(e)[:300]}") @app.post("/propisi/search_set") def propisi_search_set_post(body: PropisiSearchSetRequest): _require_propisi() top_k = _clamp_top_k(body.top_k) hybrid_top_k = _clamp_top_k(body.hybrid_top_k or top_k) clean_q, coll, query_filter = _prepare_propisi_query( body.q, body.collection, body.filter_vrsta_akta, body.filter_corpus_scope, body.filter_dbid_in, body.filter_status, body.filter_chunk_type, body.filter_year_from, ) if not clean_q: return {"hits": [], "dense": [], "sparse": [], "hybrid": [], "collection": coll} try: dense_q = nh3.clean(str(body.q_dense or body.q or "").strip()) or clean_q sparse_q = nh3.clean(str(body.q_sparse or body.q or "").strip()) or clean_q t_enc = time.perf_counter() dense_vec = _encode_propisi_dense(dense_q) sparse_query = _encode_propisi_sparse(sparse_q) encode_ms = (time.perf_counter() - t_enc) * 1000 print(f"[{encode_ms:.0f}ms] Propisi search_set encode (dense+sparse)") t_hybrid = time.perf_counter() hybrid = _with_propisi_lane_meta( _propisi_hybrid_search_internal( clean_q, coll, hybrid_top_k, query_filter, dense_vec=dense_vec, sparse_query=sparse_query, ), "hybrid", ) hybrid_lane_ms = (time.perf_counter() - t_hybrid) * 1000 hits = hybrid[:top_k] return { "hits": hits, "dense": [], "sparse": [], "hybrid": hybrid, "collection": coll, "meta": { "hybrid_top_k": hybrid_top_k, "combined": len(hits), "timings": { "encode_ms": round(encode_ms, 1), "hybrid_lane_ms": round(hybrid_lane_ms, 1), "total_ms": round(encode_ms + hybrid_lane_ms, 1), }, }, } except HTTPException: raise except Exception as e: print(f"❌ Propisi search_set error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Propisi search_set failed: {str(e)[:300]}") def _propisi_search_hybrid_response( *, q: str, top_k: int, collection: Optional[str] = None, filter_vrsta_akta: Any = None, filter_corpus_scope: Any = None, filter_dbid_in: Any = None, filter_status: Any = "Vazeci", filter_chunk_type: Any = None, filter_year_from: Optional[int] = None, ): _require_propisi() top_k = _clamp_top_k(top_k) clean_q, coll, query_filter = _prepare_propisi_query( q, collection, filter_vrsta_akta, filter_corpus_scope, filter_dbid_in, filter_status, filter_chunk_type, filter_year_from, ) if not clean_q: return {"hits": [], "collection": coll} try: return {"hits": _propisi_hybrid_search_internal(clean_q, coll, top_k, query_filter), "collection": coll} except HTTPException: raise except Exception as e: print(f"❌ Propisi hybrid search error: {e}\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Propisi hybrid search failed: {str(e)[:300]}")
"""
Combined HF Space app — Bilten + Odluke + Propisi RAG (search only, no reranking).
Bilten: SentenceTransformer (BAAI/bge-m3) + Qdrant/bm25 sparse.
Endpoints: /search.
Odluke: Shared SentenceTransformer (BAAI/bge-m3) dense + Qdrant/bm25 sparse, RRF hybrid.
Endpoints: /odluke/search_sparse, /odluke/search_hybrid,
/odluke/search_set (preferred: one encode, dense + sparse + hybrid Qdrant, merged hits).
Propisi: Shared SentenceTransformer (BAAI/bge-m3) dense + Qdrant/bm25 sparse, RRF hybrid.
Endpoints: /propisi/search_dense, /propisi/search_sparse, /propisi/search_hybrid.
Required env vars:
Bilten: QDRANT_URL, QDRANT_API_KEY, COLLECTION_NAME (default: senatus-court-opinions-v4)
Odluke: ODLUKE_QDRANT_URL, ODLUKE_QDRANT_API_KEY, ODLUKE_COLLECTION_NAME
Propisi: PROPISI_QDRANT_URL, PROPISI_QDRANT_API_KEY, PROPISI_COLLECTION_NAME
Shared: HF_TOKEN
"""
import os
# HF / Docker images sometimes set OMP_NUM_THREADS to a value libgomp rejects (empty, "auto", etc.)
# Align threads with actual core count: oversubscription (npr. OMP=8 na 2 vCPU)
# uzrokuje thrashing i SPORIJI encode. Mora se postaviti PRIJE importa torch-a.
_omp = os.environ.get("OMP_NUM_THREADS", "").strip()
if not _omp or not _omp.isdigit() or int(_omp) < 1:
os.environ["OMP_NUM_THREADS"] = str(os.cpu_count() or 4)
import re
import time
import traceback
import nh3
from fastapi import FastAPI, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import List, Optional, Dict, Any, Union
from qdrant_client import QdrantClient, models
from qdrant_client.models import FieldCondition, Filter, MatchText, MatchValue
from sentence_transformers import SentenceTransformer
from fastembed import SparseTextEmbedding
import torch
# Uskladi PyTorch CPU thread pool sa OMP-om (postavljen iznad prema cpu_count).
try:
torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "0")) or (os.cpu_count() or 4))
except Exception:
pass
def _onnx_runtime_has_cuda() -> bool:
"""HF Spaces often ship onnxruntime (CPU-only); CUDA EP requires onnxruntime-gpu matching CUDA."""
try:
import onnxruntime as ort
return "CUDAExecutionProvider" in ort.get_available_execution_providers()
except Exception:
return False
# ============================================================================
# BILTEN CONFIG
# ============================================================================
QDRANT_URL = os.environ.get("QDRANT_URL")
QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY")
HF_TOKEN = os.environ.get("HF_TOKEN")
COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "senatus-court-opinions-v4")
if not QDRANT_URL or not QDRANT_API_KEY:
print("WARNING: QDRANT_URL or QDRANT_API_KEY not set. Bilten search will return 503.")
# ============================================================================
# ODLUKE CONFIG (shared models; collection from env only)
# ============================================================================
ODLUKE_QDRANT_URL = os.environ.get("ODLUKE_QDRANT_URL") or os.environ.get("QDRANT_URL")
ODLUKE_QDRANT_API_KEY = os.environ.get("ODLUKE_QDRANT_API_KEY") or os.environ.get("QDRANT_API_KEY")
ODLUKE_COLLECTION_NAME = (os.environ.get("ODLUKE_COLLECTION_NAME") or "").strip()
ODLUKE_DENSE_VECTOR_NAME = os.environ.get("ODLUKE_DENSE_VECTOR_NAME", "dense")
ODLUKE_SPARSE_VECTOR_NAME = os.environ.get("ODLUKE_SPARSE_VECTOR_NAME", "sparse")
if not ODLUKE_QDRANT_URL or not ODLUKE_QDRANT_API_KEY:
print("WARNING: ODLUKE_QDRANT_URL or ODLUKE_QDRANT_API_KEY not set. Odluke search will return 503.")
if ODLUKE_QDRANT_URL and ODLUKE_QDRANT_API_KEY and not ODLUKE_COLLECTION_NAME:
print("WARNING: ODLUKE_COLLECTION_NAME not set. Set it in Space secrets to enable Odluke search.")
# ============================================================================
# PROPISI CONFIG (shared models; single collection)
# ============================================================================
PROPISI_QDRANT_URL = os.environ.get("PROPISI_QDRANT_URL") or os.environ.get("QDRANT_URL")
PROPISI_QDRANT_API_KEY = os.environ.get("PROPISI_QDRANT_API_KEY") or os.environ.get("QDRANT_API_KEY")
PROPISI_COLLECTION_NAME = (
os.environ.get("PROPISI_COLLECTION_NAME")
or os.environ.get("PROPISI_COLLECTION")
or "senatus-propisi-v1"
).strip()
PROPISI_DENSE_VECTOR_NAME = os.environ.get("PROPISI_DENSE_VECTOR_NAME", "dense")
PROPISI_SPARSE_VECTOR_NAME = os.environ.get("PROPISI_SPARSE_VECTOR_NAME", "sparse")
# Task 1 (synthetic-question search lane): separate pointer-only collection,
# same Qdrant cluster/client, dense-only vector named "dense". Never mixed
# into PROPISI_COLLECTION_NAME.
PROPISI_QUESTIONS_COLLECTION_NAME = (
os.environ.get("PROPISI_QUESTIONS_COLLECTION_NAME") or "senatus-propisi-questions-v1"
).strip()
# Search-time HNSW ef (recall lever). Kolekcija ima m=32/ef_construct=200, ali se
# ef pretrage nigdje nije postavljao -> default. 160 podiže recall (da tačan član