feat: Add VisionQuest Indic RAG core services, vector search, safety guardrails & web UI

Dockerfile

@@ -6,9 +6,9 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     PORT=7860 \
     HOST=0.0.0.0 \
-    DATA_DIR=/app/data \
-    INDEX_DIR=/app/data/indexes \
-    PROCESSED_DIR=/app/data/processed \
+    DATA_DIR=/app/knowledge_base \
+    INDEX_DIR=/app/knowledge_base/indexes \
+    PROCESSED_DIR=/app/knowledge_base/processed \
     HF_HOME=/app/.cache/huggingface
 
 WORKDIR /app
@@ -33,16 +33,16 @@ RUN python -c "from transformers import AutoTokenizer, AutoModel; AutoTokenizer.
 # Copy application source code and pre-built index/processed artifacts
 COPY config.py .
 COPY app.py .
-COPY data/ ./data/
-COPY chunking/ ./chunking/
-COPY retrieval/ ./retrieval/
-COPY stt/ ./stt/
-COPY guardrails/ ./guardrails/
-COPY generation/ ./generation/
-COPY pipeline/ ./pipeline/
-COPY api/ ./api/
-COPY benchmark/ ./benchmark/
-COPY demo/ ./demo/
+COPY knowledge_base/ ./knowledge_base/
+COPY doc_chunking/ ./doc_chunking/
+COPY vector_search/ ./vector_search/
+COPY voice_stt/ ./voice_stt/
+COPY safety_guardrails/ ./safety_guardrails/
+COPY llm_synthesis/ ./llm_synthesis/
+COPY rag_pipeline/ ./rag_pipeline/
+COPY api_endpoints/ ./api_endpoints/
+COPY latency_benchmarks/ ./latency_benchmarks/
+COPY web_ui/ ./web_ui/
 
 # Create user for Hugging Face Spaces security
 RUN useradd -m -u 1000 user && \
@@ -53,4 +53,4 @@ USER user
 EXPOSE 7860
 
 # Launch FastAPI server
-CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["python", "app.py"]

{api → api_endpoints}/main.py

@@ -23,10 +23,10 @@ from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 
 import config
-from pipeline.orchestrator import get_orchestrator, RAGPipelineOrchestrator
-from pipeline.schemas import QueryRequest, QueryResponse
-from retrieval.embed import get_embedder
-from retrieval.index_faiss import get_index_manager
+from rag_pipeline.orchestrator import get_orchestrator, RAGPipelineOrchestrator
+from rag_pipeline.schemas import QueryRequest, QueryResponse
+from vector_search.embed import get_embedder
+from vector_search.index_faiss import get_index_manager
 
 logger = logging.getLogger(__name__)
 

app.py

@@ -52,15 +52,15 @@ except ImportError:
             return func
 
 import config
-from pipeline.orchestrator import get_orchestrator
-from pipeline.schemas import QueryRequest, QueryResponse
-from retrieval.embed import get_embedder
-from retrieval.index_faiss import get_index_manager
+from rag_pipeline.orchestrator import get_orchestrator
+from rag_pipeline.schemas import QueryRequest, QueryResponse
+from vector_search.embed import get_embedder
+from vector_search.index_faiss import get_index_manager
 
 
 # Read the full custom HTML Command Center UI
 def get_custom_html() -> str:
-    demo_file = config.BASE_DIR / "demo" / "index.html"
+    demo_file = config.BASE_DIR / "web_ui" / "index.html"
     if demo_file.exists():
         with open(demo_file, "r", encoding="utf-8") as f:
             return f.read()
@@ -100,7 +100,7 @@ async def startup_event():
 @app.get("/", response_class=HTMLResponse)
 async def serve_index() -> HTMLResponse:
     """Serve full retro-tropical Command Center UI directly to browser."""
-    demo_file = config.BASE_DIR / "demo" / "index.html"
+    demo_file = config.BASE_DIR / "web_ui" / "index.html"
     if demo_file.exists():
         with open(demo_file, "r", encoding="utf-8") as f:
             return HTMLResponse(content=f.read())
@@ -110,7 +110,7 @@ async def serve_index() -> HTMLResponse:
 @app.get("/ironman.png")
 async def serve_ironman() -> FileResponse:
     """Serve Iron Man sprite image."""
-    img_path = config.BASE_DIR / "demo" / "ironman.png"
+    img_path = config.BASE_DIR / "web_ui" / "ironman.png"
     if img_path.exists():
         return FileResponse(img_path)
     raise HTTPException(status_code=404, detail="Iron Man image missing")
@@ -119,7 +119,7 @@ async def serve_ironman() -> FileResponse:
 @app.get("/thor.png")
 async def serve_thor() -> FileResponse:
     """Serve Thor sprite image."""
-    img_path = config.BASE_DIR / "demo" / "thor.png"
+    img_path = config.BASE_DIR / "web_ui" / "thor.png"
     if img_path.exists():
         return FileResponse(img_path)
     raise HTTPException(status_code=404, detail="Thor image missing")
@@ -128,7 +128,7 @@ async def serve_thor() -> FileResponse:
 @app.get("/cap.png")
 async def serve_cap() -> FileResponse:
     """Serve Captain America sprite image."""
-    img_path = config.BASE_DIR / "demo" / "cap.png"
+    img_path = config.BASE_DIR / "web_ui" / "cap.png"
     if img_path.exists():
         return FileResponse(img_path)
     raise HTTPException(status_code=404, detail="Captain America image missing")

config.py

@@ -68,11 +68,11 @@ def get_language_info(lang_code: str) -> dict:
 # 2. PATHS CONFIGURATION
 # ==========================================
 BASE_DIR = Path(__file__).resolve().parent
-DATA_DIR = Path(os.getenv("DATA_DIR", BASE_DIR / "data"))
+DATA_DIR = Path(os.getenv("DATA_DIR", BASE_DIR / "knowledge_base"))
 RAW_DATA_DIR = DATA_DIR / "raw"
 PROCESSED_DATA_DIR = DATA_DIR / "processed"
 INDEX_DIR = DATA_DIR / "indexes"
-BENCHMARK_RESULTS_DIR = BASE_DIR / "benchmark" / "results"
+BENCHMARK_RESULTS_DIR = BASE_DIR / "latency_benchmarks" / "results"
 
 for d in [DATA_DIR, RAW_DATA_DIR, PROCESSED_DATA_DIR, INDEX_DIR, BENCHMARK_RESULTS_DIR]:
     d.mkdir(parents=True, exist_ok=True)

{chunking → doc_chunking}/passage_native.py

@@ -6,7 +6,7 @@ Baseline, zero-loss index strategy preserving original passage boundaries and qu
 """
 
 from typing import Any, Dict, List
-from chunking.metadata import Chunk, estimate_token_count
+from doc_chunking.metadata import Chunk, estimate_token_count
 
 
 def chunk_passage_native(passage_dict: Dict[str, Any]) -> Chunk:

{chunking → doc_chunking}/semantic.py

@@ -11,7 +11,7 @@ For long documents:
 from typing import Any, Dict, List, Optional
 import numpy as np
 import config
-from chunking.metadata import (
+from doc_chunking.metadata import (
     Chunk,
     split_sentences_multilingual,
     calculate_overlap_tokens,

{chunking → doc_chunking}/sentence_window.py

@@ -9,7 +9,7 @@ For long documents:
 
 from typing import Any, Dict, List
 import config
-from chunking.metadata import (
+from doc_chunking.metadata import (
     Chunk,
     split_sentences_multilingual,
     calculate_overlap_tokens,

{data → knowledge_base}/build_corpus.py

@@ -79,7 +79,7 @@ def load_raw_dataset_for_lang(lang: str, max_queries: int = 6000) -> List[Dict[s
     else:
         # Fallback: Pull from Hugging Face
         try:
-            from datasets import load_dataset
+            from knowledge_base import load_dataset
             logger.info(f"Downloading dataset for language '{lang}' from Hugging Face...")
             if lang == "en":
                 try:

{benchmark → latency_benchmarks}/quick_eval.py

@@ -8,8 +8,8 @@ if hasattr(sys.stdout, "reconfigure"):
     sys.stdout.reconfigure(encoding="utf-8")
 
 import config
-from pipeline.orchestrator import get_orchestrator
-from pipeline.schemas import QueryRequest
+from rag_pipeline.orchestrator import get_orchestrator
+from rag_pipeline.schemas import QueryRequest
 
 async def main():
     print(f"--- RAG Pipeline Latency Benchmark ---")

{benchmark → latency_benchmarks}/run_cold_start_bench.py

@@ -31,8 +31,8 @@ if sys.platform == "win32":
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
 import config
-from pipeline.orchestrator import get_orchestrator
-from pipeline.schemas import QueryRequest
+from rag_pipeline.orchestrator import get_orchestrator
+from rag_pipeline.schemas import QueryRequest
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 logger = logging.getLogger("cold_start_bench")

{benchmark → latency_benchmarks}/run_latency_bench.py

@@ -26,8 +26,8 @@ import psutil
 # Ensure project root is in sys.path
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 import config
-from pipeline.orchestrator import get_orchestrator
-from pipeline.schemas import QueryRequest, QueryResponse
+from rag_pipeline.orchestrator import get_orchestrator
+from rag_pipeline.schemas import QueryRequest, QueryResponse
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 logger = logging.getLogger(__name__)
@@ -269,7 +269,7 @@ async def run_benchmark(iterations: int = 1) -> Dict[str, Any]:
     logger.info(f"Benchmark results successfully saved to {json_path} and {csv_path}")
     
     try:
-        from benchmark.report import generate_latency_report
+        from latency_benchmarks.report import generate_latency_report
         generate_latency_report(json_path=json_path)
         logger.info("Latency report successfully generated!")
     except Exception as e:

{benchmark → latency_benchmarks}/run_speed_bench_50.py

@@ -23,8 +23,8 @@ import psutil
 # Ensure project root is in sys.path
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 import config
-from pipeline.orchestrator import get_orchestrator
-from pipeline.schemas import QueryRequest, QueryResponse
+from rag_pipeline.orchestrator import get_orchestrator
+from rag_pipeline.schemas import QueryRequest, QueryResponse
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 logger = logging.getLogger("speed_bench_50")

{generation → llm_synthesis}/extractive.py

@@ -9,8 +9,8 @@ from typing import Any, Dict, List, Optional
 import numpy as np
 import re
 import config
-from chunking.metadata import split_sentences_multilingual
-from generation.answer_cache import get_answer_cache
+from doc_chunking.metadata import split_sentences_multilingual
+from llm_synthesis.answer_cache import get_answer_cache
 
 
 def extract_answer_from_passage(

{training → model_training}/train_colab.py

@@ -12,7 +12,7 @@ Usage in Google Colab:
 import os
 import json
 import torch
-from datasets import load_dataset
+from knowledge_base import load_dataset
 from transformers import (
     AutoModelForCausalLM,
     AutoTokenizer,

{pipeline → rag_pipeline}/orchestrator.py

@@ -17,24 +17,24 @@ from typing import Any, Dict, List, Optional
 import numpy as np
 
 import config
-from pipeline.schemas import (
+from rag_pipeline.schemas import (
     GuardrailFlags,
     QueryRequest,
     QueryResponse,
     RetrievedChunk,
     StageTiming,
 )
-from retrieval.embed import get_embedder
-from retrieval.index_faiss import get_index_manager
-from retrieval.rerank import rerank_bm25_hybrid, rerank_cross_encoder, get_cross_encoder
-from chunking.hybrid_merge import merge_and_fuse_candidates
-from guardrails.pre_retrieval import check_unsafe_content, check_off_topic_query, check_query_intent, get_safety_telemetry
-from guardrails.prompt_guard import get_prompt_guard_detector
-from guardrails.post_generation import check_grounding, DECLINED_RESPONSE_TEMPLATE
-from generation.extractive import generate_extractive
-from generation.answer_cache import get_answer_cache
-from generation.llm_fallback import get_llm_adapter
-from stt.sarvam_client import get_stt_client
+from vector_search.embed import get_embedder
+from vector_search.index_faiss import get_index_manager
+from vector_search.rerank import rerank_bm25_hybrid, rerank_cross_encoder, get_cross_encoder
+from doc_chunking.hybrid_merge import merge_and_fuse_candidates
+from safety_guardrails.pre_retrieval import check_unsafe_content, check_off_topic_query, check_query_intent, get_safety_telemetry
+from safety_guardrails.prompt_guard import get_prompt_guard_detector
+from safety_guardrails.post_generation import check_grounding, DECLINED_RESPONSE_TEMPLATE
+from llm_synthesis.extractive import generate_extractive
+from llm_synthesis.answer_cache import get_answer_cache
+from llm_synthesis.llm_fallback import get_llm_adapter
+from voice_stt.sarvam_client import get_stt_client
 
 logger = logging.getLogger(__name__)
 
@@ -550,7 +550,7 @@ class RAGPipelineOrchestrator:
         
         if config.ENABLE_LOCAL_SLM:
             # High-speed local offline SLM generation on CPU (< 60 ms)
-            from generation.local_slm import get_local_slm_adapter
+            from llm_synthesis.local_slm import get_local_slm_adapter
             context_blocks = []
             for i, c in enumerate(reranked_chunks[:3]):
                 lang_code = c.get("source_lang", "UNK").upper()