""" Global Configuration for Voice-Enabled Indic RAG System. CRITICAL EXTENSIBILITY RULE: `LANGUAGES` is the single source of truth for active languages across the entire codebase. All scripts (build_corpus.py, augment_longdocs.py, index_faiss.py, orchestrator.py, guardrails, API, etc.) MUST read dynamically from `LANGUAGES`. Extending to 13+ languages requires modifying ONLY this list. """ import os from pathlib import Path from dotenv import load_dotenv # Limit background thread creation to preserve Windows system resources os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["OMP_NUM_THREADS"] = "2" os.environ["MKL_NUM_THREADS"] = "2" # Load environment variables load_dotenv() # ========================================== # 1. LANGUAGE CONFIGURATION (Single Source of Truth) # ========================================== # Active languages for the deployed Space. Keep this list as the single source of truth. LANGUAGES = ["en", "hi", "mr"] # Comprehensive registry of supported Indic language metadata for MSMARCO-XI & STT mapping SUPPORTED_LANGUAGE_REGISTRY = { "as": {"name": "Assamese", "script": "Beng", "msmarco_file": "asm", "sarvam_code": "as-IN"}, "bn": {"name": "Bengali", "script": "Beng", "msmarco_file": "ben", "sarvam_code": "bn-IN"}, "gu": {"name": "Gujarati", "script": "Gujr", "msmarco_file": "guj", "sarvam_code": "gu-IN"}, "hi": {"name": "Hindi", "script": "Deva", "msmarco_file": "hin", "sarvam_code": "hi-IN"}, "kn": {"name": "Kannada", "script": "Knda", "msmarco_file": "kan", "sarvam_code": "kn-IN"}, "ml": {"name": "Malayalam", "script": "Mlym", "msmarco_file": "mal", "sarvam_code": "ml-IN"}, "mr": {"name": "Marathi", "script": "Deva", "msmarco_file": "mar", "sarvam_code": "mr-IN"}, "ne": {"name": "Nepali", "script": "Deva", "msmarco_file": "nep", "sarvam_code": "ne-NP"}, "or": {"name": "Odia", "script": "Orya", "msmarco_file": "ori", "sarvam_code": "od-IN"}, "pa": {"name": "Punjabi", "script": "Guru", "msmarco_file": "pan", "sarvam_code": "pa-IN"}, "sa": {"name": "Sanskrit", "script": "Deva", "msmarco_file": "san", "sarvam_code": "sa-IN"}, "ta": {"name": "Tamil", "script": "Taml", "msmarco_file": "tam", "sarvam_code": "ta-IN"}, "te": {"name": "Telugu", "script": "Telu", "msmarco_file": "tel", "sarvam_code": "te-IN"}, "ur": {"name": "Urdu", "script": "Arab", "msmarco_file": "urd", "sarvam_code": "ur-IN"}, "en": {"name": "English", "script": "Latn", "msmarco_file": "eng", "sarvam_code": "en-IN"}, } def get_language_info(lang_code: str) -> dict: """Retrieve metadata for any registered language code with safe fallback.""" if not lang_code or lang_code.lower() in ["auto", "unknown", "none", ""]: return { "name": "Auto-Detect", "script": "Unknown", "msmarco_file": "unknown", "sarvam_code": "unknown", } return SUPPORTED_LANGUAGE_REGISTRY.get( lang_code.lower(), { "name": lang_code.upper(), "script": "Unknown", "msmarco_file": lang_code, "sarvam_code": "unknown", }, ) # ========================================== # 2. PATHS CONFIGURATION # ========================================== BASE_DIR = Path(__file__).resolve().parent DATA_DIR = Path(os.getenv("DATA_DIR", BASE_DIR / "knowledge_base")) RAW_DATA_DIR = DATA_DIR / "raw" PROCESSED_DATA_DIR = DATA_DIR / "processed" INDEX_DIR = DATA_DIR / "indexes" BENCHMARK_RESULTS_DIR = BASE_DIR / "latency_benchmarks" / "results" for d in [DATA_DIR, RAW_DATA_DIR, PROCESSED_DATA_DIR, INDEX_DIR, BENCHMARK_RESULTS_DIR]: d.mkdir(parents=True, exist_ok=True) # ========================================== # 3. EMBEDDING & VECTOR RETRIEVAL CONFIG # ========================================== # intfloat/multilingual-e5-small (MUST use 'query: ' and 'passage: ' prefixes) EMBEDDING_MODEL_NAME = os.getenv("EMBEDDING_MODEL_NAME", "intfloat/multilingual-e5-small") EMBEDDING_DIM = 384 QUERY_PREFIX = "query: " PASSAGE_PREFIX = "passage: " # ONNX Runtime CPU Acceleration Settings ENABLE_ONNX_EMBEDDING = os.getenv("ENABLE_ONNX_EMBEDDING", "true").lower() == "true" ENABLE_ONNX_CROSS_ENCODER = os.getenv("ENABLE_ONNX_CROSS_ENCODER", "true").lower() == "true" ONNX_MODELS_DIR = DATA_DIR / "onnx_models" ONNX_NUM_THREADS = int(os.getenv("ONNX_NUM_THREADS", str(min(4, os.cpu_count() or 2)))) ONNX_MODELS_DIR.mkdir(parents=True, exist_ok=True) # Context Bounding & Passage Token Truncation (64 tokens for sub-200ms CPU budget) CONTEXT_BOUNDING_MAX_TOKENS = int(os.getenv("CONTEXT_BOUNDING_MAX_TOKENS", "64")) # FAISS HNSW Index Hyperparameters # Build embeddings in bounded batches so an uncapped corpus does not require # holding every tokenized batch and vector in memory at once. INDEX_BUILD_BATCH_SIZE = int(os.getenv("INDEX_BUILD_BATCH_SIZE", "512")) _MAX_PASSAGES_ENV = os.getenv("MAX_INDEX_PASSAGES_PER_LANG") if _MAX_PASSAGES_ENV is None or _MAX_PASSAGES_ENV.strip() == "": MAX_INDEX_PASSAGES_PER_LANG = None # explicit: no cap, not a silent fallback else: MAX_INDEX_PASSAGES_PER_LANG = int(_MAX_PASSAGES_ENV) # Vector Database Engine Selection ("faiss" or "qdrant") VECTOR_STORE_BACKEND = os.getenv("VECTOR_STORE_BACKEND", "faiss").lower() QDRANT_STORAGE_PATH = Path(os.getenv("QDRANT_STORAGE_PATH", DATA_DIR / "qdrant_db")) QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "") QDRANT_URL = os.getenv("QDRANT_URL", "") HNSW_M = 32 HNSW_EF_CONSTRUCTION = 200 HNSW_EF_SEARCH = 64 # Retrieval Top-K defaults FAISS_TOP_K = 15 RERANK_TOP_K = 5 HYBRID_BM25_WEIGHT = 0.35 # Dense score weight = 1 - HYBRID_BM25_WEIGHT # Cross-Encoder Re-Ranking Configuration # Default to False for ultra-fast (<2ms) Script-Aware BM25 + Dense Hybrid RRF fusion. # When enabled, utilizes INT8 Dynamic ONNX with 64-token bounding (<35ms on CPU). ENABLE_CROSS_ENCODER = os.getenv("ENABLE_CROSS_ENCODER", "false").lower() == "true" CROSS_ENCODER_MODEL_NAME = os.getenv( "CROSS_ENCODER_MODEL_NAME", "nreimers/mmarco-mMiniLMv2-L6-H384-v1" ) CROSS_ENCODER_LOCAL_CACHE = Path( os.getenv( "CROSS_ENCODER_LOCAL_CACHE", "", ) ) CROSS_ENCODER_TOP_K = int(os.getenv("CROSS_ENCODER_TOP_K", "2")) CROSS_ENCODER_THRESHOLD = float(os.getenv("CROSS_ENCODER_THRESHOLD", "0.15")) # ========================================== # 4. CHUNKING CONFIGURATION # ========================================== SENTENCE_WINDOW_SIZE = 1 # +-1 sentence window context CHUNK_OVERLAP_PERCENT = 0.15 # 15% token overlap SEMANTIC_SIMILARITY_THRESHOLD = 0.65 # Cosine distance spike threshold # ========================================== # 5. STT CONFIGURATION (Sarvam Saaras v3) # ========================================== SARVAM_API_KEY = os.getenv("SARVAM_API_KEY", "") SARVAM_MODEL = "saaras:v3" SARVAM_MODE = "transcribe" SARVAM_STT_TIMEOUT_SECONDS = 10.0 SARVAM_STT_MAX_RETRIES = 1 # ========================================== # 6. GUARDRAIL THRESHOLDS # ========================================== # Pre-retrieval off-topic cosine distance threshold from nearest corpus centroid OFF_TOPIC_DISTANCE_THRESHOLD = 0.55 # Calibrated for multilingual-e5-small normalized embeddings (1 - cosine_similarity) # Post-retrieval confidence threshold (calibrated composite dense & lexical match score) MIN_CONFIDENT_MATCH_SCORE = float(os.getenv("MIN_CONFIDENT_MATCH_SCORE", "0.35")) # Post-generation grounding check threshold (lexical + semantic overlap) GROUNDING_OVERLAP_THRESHOLD = 0.30 # Meta Prompt-Guard 86M Sub-10ms Neural Safety & Indirect Prompt Injection Guardrail ENABLE_PROMPT_GUARD = os.getenv("ENABLE_PROMPT_GUARD", "false").lower() == "true" PROMPT_GUARD_MODEL_NAME = os.getenv("PROMPT_GUARD_MODEL_NAME", "meta-llama/Prompt-Guard-86M") PROMPT_GUARD_ONNX_REPO = os.getenv("PROMPT_GUARD_ONNX_REPO", "prompt-security/Prompt-Guard-86M_onnx") PROMPT_GUARD_ONNX_PATH = ONNX_MODELS_DIR / "prompt_guard_86m.onnx" PROMPT_GUARD_THRESHOLD = float(os.getenv("PROMPT_GUARD_THRESHOLD", "0.5")) PROMPT_GUARD_TEMPERATURE = float(os.getenv("PROMPT_GUARD_TEMPERATURE", "1.0")) ENABLE_CONTEXT_CHUNK_SCAN = os.getenv("ENABLE_CONTEXT_CHUNK_SCAN", "true").lower() == "true" # Pre-retrieval Query Intent Guardrail (Filters creative writing, personal advice, planning, roleplay) ENABLE_QUERY_INTENT_FILTER = os.getenv("ENABLE_QUERY_INTENT_FILTER", "true").lower() == "true" # ========================================== # 7. LLM MULTI-TIER PROVIDER & GENERATION CONFIG # ========================================== # HARD OVERRIDE: Prevent live network calls during critical path latency budget (<200ms) # Setting this to False keeps all LLM/Groq/Cerebras code dormant even if API keys are present. ALLOW_NETWORK_CALLS_IN_PIPELINE = os.getenv("ALLOW_NETWORK_CALLS_IN_PIPELINE", "true").lower() == "true" # Semantic Answer Cache (Fast lookup for gold answers of known queries in MSMARCO) SEMANTIC_ANSWER_CACHE_ENABLED = True SEMANTIC_ANSWER_CACHE_THRESHOLD = 0.93 # Dynamic In-Memory Vector LRU Semantic Cache (Tier-1 Hot Cache for all queries) DYNAMIC_SEMANTIC_CACHE_ENABLED = os.getenv("DYNAMIC_SEMANTIC_CACHE_ENABLED", "true").lower() == "true" DYNAMIC_SEMANTIC_CACHE_MAX_ENTRIES = int(os.getenv("DYNAMIC_SEMANTIC_CACHE_MAX_ENTRIES", "2048")) DYNAMIC_SEMANTIC_CACHE_THRESHOLD = float(os.getenv("DYNAMIC_SEMANTIC_CACHE_THRESHOLD", "0.92")) # Tier-1 Primary: Groq High-Speed Llama-3.3 / Mixtral / GPT-OSS API (~150ms) GROQ_API_KEY = os.getenv("GROQ_API_KEY", os.getenv("LLM_API_KEY", "")) LLM_API_KEY = os.getenv("LLM_API_KEY", GROQ_API_KEY) LLM_BASE_URL = os.getenv("LLM_BASE_URL", "https://api.groq.com/openai/v1") LLM_MODEL = os.getenv("LLM_MODEL", "openai/gpt-oss-120b") LLM_TIMEOUT_SECONDS = 15.0 # Tier-2 & Tier-3 Backup: Cerebras High-Speed LPU (120B model for high instruction following) CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY", "") CEREBRAS_BASE_URL = os.getenv("CEREBRAS_BASE_URL", "https://api.cerebras.ai/v1") CEREBRAS_MODEL = os.getenv("CEREBRAS_MODEL", "gpt-oss-120b") CEREBRAS_FALLBACK_MODEL = os.getenv("CEREBRAS_FALLBACK_MODEL", "gemma-4-31b") CEREBRAS_TIMEOUT_SECONDS = 12.0 # Local Small Language Model (SLM) Offline Generation (Sub-100ms on CPU) ENABLE_LOCAL_SLM = os.getenv("ENABLE_LOCAL_SLM", "false").lower() == "true" LOCAL_SLM_MODEL_PATH = os.getenv("LOCAL_SLM_MODEL_PATH", "Qwen/Qwen2.5-0.5B-Instruct") ADAPTION_API_KEY = os.getenv("ADAPTION_API_KEY", "") # ========================================== # 8. SERVER CONFIGURATION # ========================================== HOST = os.getenv("HOST", "0.0.0.0") PORT = int(os.getenv("PORT", "7860")) REQUEST_TIMEOUT_SECONDS = float(os.getenv("REQUEST_TIMEOUT_SECONDS", "15.0"))