config.py

"""
Global Configuration for Voice-Enabled Indic RAG System.

CRITICAL EXTENSIBILITY RULE:
`LANGUAGES` is the single source of truth for active languages across the entire codebase.
All scripts (build_corpus.py, augment_longdocs.py, index_faiss.py, orchestrator.py,
guardrails, API, etc.) MUST read dynamically from `LANGUAGES`.
Extending to 13+ languages requires modifying ONLY this list.
"""

import os
from pathlib import Path
from dotenv import load_dotenv

# Limit background thread creation to preserve Windows system resources
os.environ["TOKENIZERS_PARALLELISM"] = "false"
os.environ["OMP_NUM_THREADS"] = "2"
os.environ["MKL_NUM_THREADS"] = "2"

# Load environment variables
load_dotenv()

# ==========================================
# 1. LANGUAGE CONFIGURATION (Single Source of Truth)
# ==========================================
# Active languages for the deployed Space. Keep this list as the single source of truth.
LANGUAGES = ["en", "hi", "mr"]

# Comprehensive registry of supported Indic language metadata for MSMARCO-XI & STT mapping
SUPPORTED_LANGUAGE_REGISTRY = {
    "as": {"name": "Assamese", "script": "Beng", "msmarco_file": "asm", "sarvam_code": "as-IN"},
    "bn": {"name": "Bengali", "script": "Beng", "msmarco_file": "ben", "sarvam_code": "bn-IN"},
    "gu": {"name": "Gujarati", "script": "Gujr", "msmarco_file": "guj", "sarvam_code": "gu-IN"},
    "hi": {"name": "Hindi", "script": "Deva", "msmarco_file": "hin", "sarvam_code": "hi-IN"},
    "kn": {"name": "Kannada", "script": "Knda", "msmarco_file": "kan", "sarvam_code": "kn-IN"},
    "ml": {"name": "Malayalam", "script": "Mlym", "msmarco_file": "mal", "sarvam_code": "ml-IN"},
    "mr": {"name": "Marathi", "script": "Deva", "msmarco_file": "mar", "sarvam_code": "mr-IN"},
    "ne": {"name": "Nepali", "script": "Deva", "msmarco_file": "nep", "sarvam_code": "ne-NP"},
    "or": {"name": "Odia", "script": "Orya", "msmarco_file": "ori", "sarvam_code": "od-IN"},
    "pa": {"name": "Punjabi", "script": "Guru", "msmarco_file": "pan", "sarvam_code": "pa-IN"},
    "sa": {"name": "Sanskrit", "script": "Deva", "msmarco_file": "san", "sarvam_code": "sa-IN"},
    "ta": {"name": "Tamil", "script": "Taml", "msmarco_file": "tam", "sarvam_code": "ta-IN"},
    "te": {"name": "Telugu", "script": "Telu", "msmarco_file": "tel", "sarvam_code": "te-IN"},
    "ur": {"name": "Urdu", "script": "Arab", "msmarco_file": "urd", "sarvam_code": "ur-IN"},
    "en": {"name": "English", "script": "Latn", "msmarco_file": "eng", "sarvam_code": "en-IN"},
}

def get_language_info(lang_code: str) -> dict:
    """Retrieve metadata for any registered language code with safe fallback."""
    if not lang_code or lang_code.lower() in ["auto", "unknown", "none", ""]:
        return {
            "name": "Auto-Detect",
            "script": "Unknown",
            "msmarco_file": "unknown",
            "sarvam_code": "unknown",
        }
    return SUPPORTED_LANGUAGE_REGISTRY.get(
        lang_code.lower(),
        {
            "name": lang_code.upper(),
            "script": "Unknown",
            "msmarco_file": lang_code,
            "sarvam_code": "unknown",
        },
    )

# ==========================================
# 2. PATHS CONFIGURATION
# ==========================================
BASE_DIR = Path(__file__).resolve().parent
DATA_DIR = Path(os.getenv("DATA_DIR", BASE_DIR / "knowledge_base"))
RAW_DATA_DIR = DATA_DIR / "raw"
PROCESSED_DATA_DIR = DATA_DIR / "processed"
INDEX_DIR = DATA_DIR / "indexes"
BENCHMARK_RESULTS_DIR = BASE_DIR / "latency_benchmarks" / "results"

for d in [DATA_DIR, RAW_DATA_DIR, PROCESSED_DATA_DIR, INDEX_DIR, BENCHMARK_RESULTS_DIR]:
    d.mkdir(parents=True, exist_ok=True)

# ==========================================
# 3. EMBEDDING & VECTOR RETRIEVAL CONFIG
# ==========================================
# intfloat/multilingual-e5-small (MUST use 'query: ' and 'passage: ' prefixes)
EMBEDDING_MODEL_NAME = os.getenv("EMBEDDING_MODEL_NAME", "intfloat/multilingual-e5-small")
EMBEDDING_DIM = 384
QUERY_PREFIX = "query: "
PASSAGE_PREFIX = "passage: "

# ONNX Runtime CPU Acceleration Settings
ENABLE_ONNX_EMBEDDING = os.getenv("ENABLE_ONNX_EMBEDDING", "true").lower() == "true"
ENABLE_ONNX_CROSS_ENCODER = os.getenv("ENABLE_ONNX_CROSS_ENCODER", "true").lower() == "true"
ONNX_MODELS_DIR = DATA_DIR / "onnx_models"
ONNX_NUM_THREADS = int(os.getenv("ONNX_NUM_THREADS", str(min(4, os.cpu_count() or 2))))
ONNX_MODELS_DIR.mkdir(parents=True, exist_ok=True)

# Context Bounding & Passage Token Truncation (64 tokens for sub-200ms CPU budget)
CONTEXT_BOUNDING_MAX_TOKENS = int(os.getenv("CONTEXT_BOUNDING_MAX_TOKENS", "64"))

# FAISS HNSW Index Hyperparameters
# Build embeddings in bounded batches so an uncapped corpus does not require
# holding every tokenized batch and vector in memory at once.
INDEX_BUILD_BATCH_SIZE = int(os.getenv("INDEX_BUILD_BATCH_SIZE", "512"))
_MAX_PASSAGES_ENV = os.getenv("MAX_INDEX_PASSAGES_PER_LANG")
if _MAX_PASSAGES_ENV is None or _MAX_PASSAGES_ENV.strip() == "":
    MAX_INDEX_PASSAGES_PER_LANG = None  # explicit: no cap, not a silent fallback
else:
    MAX_INDEX_PASSAGES_PER_LANG = int(_MAX_PASSAGES_ENV)
# Vector Database Engine Selection ("faiss" or "qdrant")
VECTOR_STORE_BACKEND = os.getenv("VECTOR_STORE_BACKEND", "faiss").lower()
QDRANT_STORAGE_PATH = Path(os.getenv("QDRANT_STORAGE_PATH", DATA_DIR / "qdrant_db"))
QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "")
QDRANT_URL = os.getenv("QDRANT_URL", "")

HNSW_M = 32
HNSW_EF_CONSTRUCTION = 200
HNSW_EF_SEARCH = 64

# Retrieval Top-K defaults
FAISS_TOP_K = 15
RERANK_TOP_K = 5
HYBRID_BM25_WEIGHT = 0.35  # Dense score weight = 1 - HYBRID_BM25_WEIGHT

# Cross-Encoder Re-Ranking Configuration
# Default to False for ultra-fast (<2ms) Script-Aware BM25 + Dense Hybrid RRF fusion.
# When enabled, utilizes INT8 Dynamic ONNX with 64-token bounding (<35ms on CPU).
ENABLE_CROSS_ENCODER = os.getenv("ENABLE_CROSS_ENCODER", "false").lower() == "true"
CROSS_ENCODER_MODEL_NAME = os.getenv(
    "CROSS_ENCODER_MODEL_NAME", "nreimers/mmarco-mMiniLMv2-L6-H384-v1"
)
CROSS_ENCODER_LOCAL_CACHE = Path(
    os.getenv(
        "CROSS_ENCODER_LOCAL_CACHE",
        "",
    )
)
CROSS_ENCODER_TOP_K = int(os.getenv("CROSS_ENCODER_TOP_K", "2"))
CROSS_ENCODER_THRESHOLD = float(os.getenv("CROSS_ENCODER_THRESHOLD", "0.15"))


# ==========================================
# 4. CHUNKING CONFIGURATION
# ==========================================
SENTENCE_WINDOW_SIZE = 1  # +-1 sentence window context
CHUNK_OVERLAP_PERCENT = 0.15  # 15% token overlap
SEMANTIC_SIMILARITY_THRESHOLD = 0.65  # Cosine distance spike threshold

# ==========================================
# 5. STT CONFIGURATION (Sarvam Saaras v3)
# ==========================================
SARVAM_API_KEY = os.getenv("SARVAM_API_KEY", "")
SARVAM_MODEL = "saaras:v3"
SARVAM_MODE = "transcribe"
SARVAM_STT_TIMEOUT_SECONDS = 10.0
SARVAM_STT_MAX_RETRIES = 1

# ==========================================
# 6. GUARDRAIL THRESHOLDS
# ==========================================
# Pre-retrieval off-topic cosine distance threshold from nearest corpus centroid
OFF_TOPIC_DISTANCE_THRESHOLD = 0.55  # Calibrated for multilingual-e5-small normalized embeddings (1 - cosine_similarity)

# Post-retrieval confidence threshold (calibrated composite dense & lexical match score)
MIN_CONFIDENT_MATCH_SCORE = float(os.getenv("MIN_CONFIDENT_MATCH_SCORE", "0.35"))





# Post-generation grounding check threshold (lexical + semantic overlap)
GROUNDING_OVERLAP_THRESHOLD = 0.30

# Meta Prompt-Guard 86M Sub-10ms Neural Safety & Indirect Prompt Injection Guardrail
ENABLE_PROMPT_GUARD = os.getenv("ENABLE_PROMPT_GUARD", "false").lower() == "true"
PROMPT_GUARD_MODEL_NAME = os.getenv("PROMPT_GUARD_MODEL_NAME", "meta-llama/Prompt-Guard-86M")
PROMPT_GUARD_ONNX_REPO = os.getenv("PROMPT_GUARD_ONNX_REPO", "prompt-security/Prompt-Guard-86M_onnx")
PROMPT_GUARD_ONNX_PATH = ONNX_MODELS_DIR / "prompt_guard_86m.onnx"
PROMPT_GUARD_THRESHOLD = float(os.getenv("PROMPT_GUARD_THRESHOLD", "0.5"))
PROMPT_GUARD_TEMPERATURE = float(os.getenv("PROMPT_GUARD_TEMPERATURE", "1.0"))
ENABLE_CONTEXT_CHUNK_SCAN = os.getenv("ENABLE_CONTEXT_CHUNK_SCAN", "true").lower() == "true"

# Pre-retrieval Query Intent Guardrail (Filters creative writing, personal advice, planning, roleplay)
ENABLE_QUERY_INTENT_FILTER = os.getenv("ENABLE_QUERY_INTENT_FILTER", "true").lower() == "true"

# ==========================================
# 7. LLM MULTI-TIER PROVIDER & GENERATION CONFIG
# ==========================================
# HARD OVERRIDE: Prevent live network calls during critical path latency budget (<200ms)
# Setting this to False keeps all LLM/Groq/Cerebras code dormant even if API keys are present.
ALLOW_NETWORK_CALLS_IN_PIPELINE = os.getenv("ALLOW_NETWORK_CALLS_IN_PIPELINE", "true").lower() == "true"

# Semantic Answer Cache (Fast lookup for gold answers of known queries in MSMARCO)
SEMANTIC_ANSWER_CACHE_ENABLED = True
SEMANTIC_ANSWER_CACHE_THRESHOLD = 0.93

# Dynamic In-Memory Vector LRU Semantic Cache (Tier-1 Hot Cache for all queries)
DYNAMIC_SEMANTIC_CACHE_ENABLED = os.getenv("DYNAMIC_SEMANTIC_CACHE_ENABLED", "true").lower() == "true"
DYNAMIC_SEMANTIC_CACHE_MAX_ENTRIES = int(os.getenv("DYNAMIC_SEMANTIC_CACHE_MAX_ENTRIES", "2048"))
DYNAMIC_SEMANTIC_CACHE_THRESHOLD = float(os.getenv("DYNAMIC_SEMANTIC_CACHE_THRESHOLD", "0.92"))

# Tier-1 Primary: Groq High-Speed Llama-3.3 / Mixtral / GPT-OSS API (~150ms)
GROQ_API_KEY = os.getenv("GROQ_API_KEY", os.getenv("LLM_API_KEY", ""))
LLM_API_KEY = os.getenv("LLM_API_KEY", GROQ_API_KEY)
LLM_BASE_URL = os.getenv("LLM_BASE_URL", "https://api.groq.com/openai/v1")
LLM_MODEL = os.getenv("LLM_MODEL", "openai/gpt-oss-120b")
LLM_TIMEOUT_SECONDS = 15.0

# Tier-2 & Tier-3 Backup: Cerebras High-Speed LPU (120B model for high instruction following)
CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY", "")
CEREBRAS_BASE_URL = os.getenv("CEREBRAS_BASE_URL", "https://api.cerebras.ai/v1")
CEREBRAS_MODEL = os.getenv("CEREBRAS_MODEL", "gpt-oss-120b")
CEREBRAS_FALLBACK_MODEL = os.getenv("CEREBRAS_FALLBACK_MODEL", "gemma-4-31b")
CEREBRAS_TIMEOUT_SECONDS = 12.0

# Local Small Language Model (SLM) Offline Generation (Sub-100ms on CPU)
ENABLE_LOCAL_SLM = os.getenv("ENABLE_LOCAL_SLM", "false").lower() == "true"
LOCAL_SLM_MODEL_PATH = os.getenv("LOCAL_SLM_MODEL_PATH", "Qwen/Qwen2.5-0.5B-Instruct")
ADAPTION_API_KEY = os.getenv("ADAPTION_API_KEY", "")

# ==========================================
# 8. SERVER CONFIGURATION
# ==========================================
HOST = os.getenv("HOST", "0.0.0.0")
PORT = int(os.getenv("PORT", "7860"))
REQUEST_TIMEOUT_SECONDS = float(os.getenv("REQUEST_TIMEOUT_SECONDS", "15.0"))