diff --git a/Dockerfile b/Dockerfile index 917eb515e49a7a91614d81a5a9d92c4c5b143103..bbe179f9cb0b10ab48d97d9309d0bf8a34087342 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,9 @@ ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ PORT=7860 \ HOST=0.0.0.0 \ - DATA_DIR=/app/data \ - INDEX_DIR=/app/data/indexes \ - PROCESSED_DIR=/app/data/processed \ + DATA_DIR=/app/knowledge_base \ + INDEX_DIR=/app/knowledge_base/indexes \ + PROCESSED_DIR=/app/knowledge_base/processed \ HF_HOME=/app/.cache/huggingface WORKDIR /app @@ -33,16 +33,16 @@ RUN python -c "from transformers import AutoTokenizer, AutoModel; AutoTokenizer. # Copy application source code and pre-built index/processed artifacts COPY config.py . COPY app.py . -COPY data/ ./data/ -COPY chunking/ ./chunking/ -COPY retrieval/ ./retrieval/ -COPY stt/ ./stt/ -COPY guardrails/ ./guardrails/ -COPY generation/ ./generation/ -COPY pipeline/ ./pipeline/ -COPY api/ ./api/ -COPY benchmark/ ./benchmark/ -COPY demo/ ./demo/ +COPY knowledge_base/ ./knowledge_base/ +COPY doc_chunking/ ./doc_chunking/ +COPY vector_search/ ./vector_search/ +COPY voice_stt/ ./voice_stt/ +COPY safety_guardrails/ ./safety_guardrails/ +COPY llm_synthesis/ ./llm_synthesis/ +COPY rag_pipeline/ ./rag_pipeline/ +COPY api_endpoints/ ./api_endpoints/ +COPY latency_benchmarks/ ./latency_benchmarks/ +COPY web_ui/ ./web_ui/ # Create user for Hugging Face Spaces security RUN useradd -m -u 1000 user && \ @@ -53,4 +53,4 @@ USER user EXPOSE 7860 # Launch FastAPI server -CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "7860"] +CMD ["python", "app.py"] diff --git a/api/main.py b/api_endpoints/main.py similarity index 96% rename from api/main.py rename to api_endpoints/main.py index 70d3a41bd912816228cb2929bf4e2a5d82e73ba5..620b4273df37644110b52752c77fe0bbaf27717a 100644 --- a/api/main.py +++ b/api_endpoints/main.py @@ -23,10 +23,10 @@ from fastapi.responses import HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles import config -from pipeline.orchestrator import get_orchestrator, RAGPipelineOrchestrator -from pipeline.schemas import QueryRequest, QueryResponse -from retrieval.embed import get_embedder -from retrieval.index_faiss import get_index_manager +from rag_pipeline.orchestrator import get_orchestrator, RAGPipelineOrchestrator +from rag_pipeline.schemas import QueryRequest, QueryResponse +from vector_search.embed import get_embedder +from vector_search.index_faiss import get_index_manager logger = logging.getLogger(__name__) diff --git a/app.py b/app.py index 76e489df7054a132fd704db4caa0ea5f17ee077c..5eb998e623350c6f8c2cc4a46f288e187892f0e2 100644 --- a/app.py +++ b/app.py @@ -52,15 +52,15 @@ except ImportError: return func import config -from pipeline.orchestrator import get_orchestrator -from pipeline.schemas import QueryRequest, QueryResponse -from retrieval.embed import get_embedder -from retrieval.index_faiss import get_index_manager +from rag_pipeline.orchestrator import get_orchestrator +from rag_pipeline.schemas import QueryRequest, QueryResponse +from vector_search.embed import get_embedder +from vector_search.index_faiss import get_index_manager # Read the full custom HTML Command Center UI def get_custom_html() -> str: - demo_file = config.BASE_DIR / "demo" / "index.html" + demo_file = config.BASE_DIR / "web_ui" / "index.html" if demo_file.exists(): with open(demo_file, "r", encoding="utf-8") as f: return f.read() @@ -100,7 +100,7 @@ async def startup_event(): @app.get("/", response_class=HTMLResponse) async def serve_index() -> HTMLResponse: """Serve full retro-tropical Command Center UI directly to browser.""" - demo_file = config.BASE_DIR / "demo" / "index.html" + demo_file = config.BASE_DIR / "web_ui" / "index.html" if demo_file.exists(): with open(demo_file, "r", encoding="utf-8") as f: return HTMLResponse(content=f.read()) @@ -110,7 +110,7 @@ async def serve_index() -> HTMLResponse: @app.get("/ironman.png") async def serve_ironman() -> FileResponse: """Serve Iron Man sprite image.""" - img_path = config.BASE_DIR / "demo" / "ironman.png" + img_path = config.BASE_DIR / "web_ui" / "ironman.png" if img_path.exists(): return FileResponse(img_path) raise HTTPException(status_code=404, detail="Iron Man image missing") @@ -119,7 +119,7 @@ async def serve_ironman() -> FileResponse: @app.get("/thor.png") async def serve_thor() -> FileResponse: """Serve Thor sprite image.""" - img_path = config.BASE_DIR / "demo" / "thor.png" + img_path = config.BASE_DIR / "web_ui" / "thor.png" if img_path.exists(): return FileResponse(img_path) raise HTTPException(status_code=404, detail="Thor image missing") @@ -128,7 +128,7 @@ async def serve_thor() -> FileResponse: @app.get("/cap.png") async def serve_cap() -> FileResponse: """Serve Captain America sprite image.""" - img_path = config.BASE_DIR / "demo" / "cap.png" + img_path = config.BASE_DIR / "web_ui" / "cap.png" if img_path.exists(): return FileResponse(img_path) raise HTTPException(status_code=404, detail="Captain America image missing") diff --git a/config.py b/config.py index c753366715c626c4c0fc6951f3347eb38f06f550..37bc8f4c75960480f794947cf5b728f993fb140e 100644 --- a/config.py +++ b/config.py @@ -68,11 +68,11 @@ def get_language_info(lang_code: str) -> dict: # 2. PATHS CONFIGURATION # ========================================== BASE_DIR = Path(__file__).resolve().parent -DATA_DIR = Path(os.getenv("DATA_DIR", BASE_DIR / "data")) +DATA_DIR = Path(os.getenv("DATA_DIR", BASE_DIR / "knowledge_base")) RAW_DATA_DIR = DATA_DIR / "raw" PROCESSED_DATA_DIR = DATA_DIR / "processed" INDEX_DIR = DATA_DIR / "indexes" -BENCHMARK_RESULTS_DIR = BASE_DIR / "benchmark" / "results" +BENCHMARK_RESULTS_DIR = BASE_DIR / "latency_benchmarks" / "results" for d in [DATA_DIR, RAW_DATA_DIR, PROCESSED_DATA_DIR, INDEX_DIR, BENCHMARK_RESULTS_DIR]: d.mkdir(parents=True, exist_ok=True) diff --git a/chunking/hybrid_merge.py b/doc_chunking/hybrid_merge.py similarity index 100% rename from chunking/hybrid_merge.py rename to doc_chunking/hybrid_merge.py diff --git a/chunking/metadata.py b/doc_chunking/metadata.py similarity index 100% rename from chunking/metadata.py rename to doc_chunking/metadata.py diff --git a/chunking/passage_native.py b/doc_chunking/passage_native.py similarity index 95% rename from chunking/passage_native.py rename to doc_chunking/passage_native.py index 20961cf400a68378819af0dcc66a2a9baa61c2c3..ee23c3aa29f1cf54b8aefa16912eabdbed3b3d07 100644 --- a/chunking/passage_native.py +++ b/doc_chunking/passage_native.py @@ -6,7 +6,7 @@ Baseline, zero-loss index strategy preserving original passage boundaries and qu """ from typing import Any, Dict, List -from chunking.metadata import Chunk, estimate_token_count +from doc_chunking.metadata import Chunk, estimate_token_count def chunk_passage_native(passage_dict: Dict[str, Any]) -> Chunk: diff --git a/chunking/semantic.py b/doc_chunking/semantic.py similarity index 99% rename from chunking/semantic.py rename to doc_chunking/semantic.py index 35bbaa610eee6a78380195a5b9382d164e2401dd..737ac841e786f4f6a994b0766d0b7a9c462f2640 100644 --- a/chunking/semantic.py +++ b/doc_chunking/semantic.py @@ -11,7 +11,7 @@ For long documents: from typing import Any, Dict, List, Optional import numpy as np import config -from chunking.metadata import ( +from doc_chunking.metadata import ( Chunk, split_sentences_multilingual, calculate_overlap_tokens, diff --git a/chunking/sentence_window.py b/doc_chunking/sentence_window.py similarity index 98% rename from chunking/sentence_window.py rename to doc_chunking/sentence_window.py index b1fe2c9ea552f86dd9da28cc01156634683963ce..241a31bc47885b650c600856d756219b8f8a1cc7 100644 --- a/chunking/sentence_window.py +++ b/doc_chunking/sentence_window.py @@ -9,7 +9,7 @@ For long documents: from typing import Any, Dict, List import config -from chunking.metadata import ( +from doc_chunking.metadata import ( Chunk, split_sentences_multilingual, calculate_overlap_tokens, diff --git a/data/augment_longdocs.py b/knowledge_base/augment_longdocs.py similarity index 100% rename from data/augment_longdocs.py rename to knowledge_base/augment_longdocs.py diff --git a/data/build_all_15_corpora.py b/knowledge_base/build_all_15_corpora.py similarity index 100% rename from data/build_all_15_corpora.py rename to knowledge_base/build_all_15_corpora.py diff --git a/data/build_corpus.py b/knowledge_base/build_corpus.py similarity index 99% rename from data/build_corpus.py rename to knowledge_base/build_corpus.py index b5204268940e7b8df90493792202ca253998fc7f..132183926535fdb0b3ef40645b45633ad915e00d 100644 --- a/data/build_corpus.py +++ b/knowledge_base/build_corpus.py @@ -79,7 +79,7 @@ def load_raw_dataset_for_lang(lang: str, max_queries: int = 6000) -> List[Dict[s else: # Fallback: Pull from Hugging Face try: - from datasets import load_dataset + from knowledge_base import load_dataset logger.info(f"Downloading dataset for language '{lang}' from Hugging Face...") if lang == "en": try: diff --git a/data/indexes/centroids.json b/knowledge_base/indexes/centroids.json similarity index 100% rename from data/indexes/centroids.json rename to knowledge_base/indexes/centroids.json diff --git a/data/indexes/index_manifest.json b/knowledge_base/indexes/index_manifest.json similarity index 100% rename from data/indexes/index_manifest.json rename to knowledge_base/indexes/index_manifest.json diff --git a/data/indexes/passage_native.faiss b/knowledge_base/indexes/passage_native.faiss similarity index 100% rename from data/indexes/passage_native.faiss rename to knowledge_base/indexes/passage_native.faiss diff --git a/data/indexes/passage_native_meta.json b/knowledge_base/indexes/passage_native_meta.json similarity index 100% rename from data/indexes/passage_native_meta.json rename to knowledge_base/indexes/passage_native_meta.json diff --git a/data/indexes/semantic_longdoc.faiss b/knowledge_base/indexes/semantic_longdoc.faiss similarity index 100% rename from data/indexes/semantic_longdoc.faiss rename to knowledge_base/indexes/semantic_longdoc.faiss diff --git a/data/indexes/semantic_longdoc_meta.json b/knowledge_base/indexes/semantic_longdoc_meta.json similarity index 100% rename from data/indexes/semantic_longdoc_meta.json rename to knowledge_base/indexes/semantic_longdoc_meta.json diff --git a/data/movie_data.py b/knowledge_base/movie_data.py similarity index 100% rename from data/movie_data.py rename to knowledge_base/movie_data.py diff --git a/data/processed/en_corpus.jsonl b/knowledge_base/processed/en_corpus.jsonl similarity index 100% rename from data/processed/en_corpus.jsonl rename to knowledge_base/processed/en_corpus.jsonl diff --git a/data/processed/en_longdocs.jsonl b/knowledge_base/processed/en_longdocs.jsonl similarity index 100% rename from data/processed/en_longdocs.jsonl rename to knowledge_base/processed/en_longdocs.jsonl diff --git a/data/processed/hi_corpus.jsonl b/knowledge_base/processed/hi_corpus.jsonl similarity index 100% rename from data/processed/hi_corpus.jsonl rename to knowledge_base/processed/hi_corpus.jsonl diff --git a/data/processed/hi_longdocs.jsonl b/knowledge_base/processed/hi_longdocs.jsonl similarity index 100% rename from data/processed/hi_longdocs.jsonl rename to knowledge_base/processed/hi_longdocs.jsonl diff --git a/data/processed/mr_corpus.jsonl b/knowledge_base/processed/mr_corpus.jsonl similarity index 100% rename from data/processed/mr_corpus.jsonl rename to knowledge_base/processed/mr_corpus.jsonl diff --git a/data/processed/mr_longdocs.jsonl b/knowledge_base/processed/mr_longdocs.jsonl similarity index 100% rename from data/processed/mr_longdocs.jsonl rename to knowledge_base/processed/mr_longdocs.jsonl diff --git a/data/tech_data.py b/knowledge_base/tech_data.py similarity index 100% rename from data/tech_data.py rename to knowledge_base/tech_data.py diff --git a/benchmark/quick_eval.py b/latency_benchmarks/quick_eval.py similarity index 94% rename from benchmark/quick_eval.py rename to latency_benchmarks/quick_eval.py index 7dbe586697a290f841a578c5b783b9ab42ba78a2..36989e65ce7a95e666ed810bf1bc9227feb111f8 100644 --- a/benchmark/quick_eval.py +++ b/latency_benchmarks/quick_eval.py @@ -8,8 +8,8 @@ if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") import config -from pipeline.orchestrator import get_orchestrator -from pipeline.schemas import QueryRequest +from rag_pipeline.orchestrator import get_orchestrator +from rag_pipeline.schemas import QueryRequest async def main(): print(f"--- RAG Pipeline Latency Benchmark ---") diff --git a/benchmark/report.py b/latency_benchmarks/report.py similarity index 100% rename from benchmark/report.py rename to latency_benchmarks/report.py diff --git a/benchmark/results/cold_start_benchmark_results.json b/latency_benchmarks/results/cold_start_benchmark_results.json similarity index 100% rename from benchmark/results/cold_start_benchmark_results.json rename to latency_benchmarks/results/cold_start_benchmark_results.json diff --git a/benchmark/results/latency_report.md b/latency_benchmarks/results/latency_report.md similarity index 100% rename from benchmark/results/latency_report.md rename to latency_benchmarks/results/latency_report.md diff --git a/benchmark/results/latency_results.csv b/latency_benchmarks/results/latency_results.csv similarity index 100% rename from benchmark/results/latency_results.csv rename to latency_benchmarks/results/latency_results.csv diff --git a/benchmark/results/latency_results.json b/latency_benchmarks/results/latency_results.json similarity index 100% rename from benchmark/results/latency_results.json rename to latency_benchmarks/results/latency_results.json diff --git a/benchmark/results/retrieval_benchmark_report.md b/latency_benchmarks/results/retrieval_benchmark_report.md similarity index 100% rename from benchmark/results/retrieval_benchmark_report.md rename to latency_benchmarks/results/retrieval_benchmark_report.md diff --git a/benchmark/results/speed_bench_50_report.md b/latency_benchmarks/results/speed_bench_50_report.md similarity index 100% rename from benchmark/results/speed_bench_50_report.md rename to latency_benchmarks/results/speed_bench_50_report.md diff --git a/benchmark/results/speed_bench_50_results.json b/latency_benchmarks/results/speed_bench_50_results.json similarity index 100% rename from benchmark/results/speed_bench_50_results.json rename to latency_benchmarks/results/speed_bench_50_results.json diff --git a/benchmark/run_cold_start_bench.py b/latency_benchmarks/run_cold_start_bench.py similarity index 98% rename from benchmark/run_cold_start_bench.py rename to latency_benchmarks/run_cold_start_bench.py index 5e130ef51dc899b187c008ec12eec89d9ac705cc..719ca3b2ed447c4538a9e9e8016faf4c97a54d7e 100644 --- a/benchmark/run_cold_start_bench.py +++ b/latency_benchmarks/run_cold_start_bench.py @@ -31,8 +31,8 @@ if sys.platform == "win32": sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from pipeline.orchestrator import get_orchestrator -from pipeline.schemas import QueryRequest +from rag_pipeline.orchestrator import get_orchestrator +from rag_pipeline.schemas import QueryRequest logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("cold_start_bench") diff --git a/benchmark/run_latency_bench.py b/latency_benchmarks/run_latency_bench.py similarity index 99% rename from benchmark/run_latency_bench.py rename to latency_benchmarks/run_latency_bench.py index 3def2154b3619d34240382439f505742ba0d3da0..e90c587d02540cbb294b5f23cadcfac666063dd4 100644 --- a/benchmark/run_latency_bench.py +++ b/latency_benchmarks/run_latency_bench.py @@ -26,8 +26,8 @@ import psutil # Ensure project root is in sys.path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from pipeline.orchestrator import get_orchestrator -from pipeline.schemas import QueryRequest, QueryResponse +from rag_pipeline.orchestrator import get_orchestrator +from rag_pipeline.schemas import QueryRequest, QueryResponse logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) @@ -269,7 +269,7 @@ async def run_benchmark(iterations: int = 1) -> Dict[str, Any]: logger.info(f"Benchmark results successfully saved to {json_path} and {csv_path}") try: - from benchmark.report import generate_latency_report + from latency_benchmarks.report import generate_latency_report generate_latency_report(json_path=json_path) logger.info("Latency report successfully generated!") except Exception as e: diff --git a/benchmark/run_speed_bench_50.py b/latency_benchmarks/run_speed_bench_50.py similarity index 99% rename from benchmark/run_speed_bench_50.py rename to latency_benchmarks/run_speed_bench_50.py index e2b3a985cdd725ca5b53b5a6bff570d05b979cbf..754dab25dc3b85c4fd4454817fb928679bc19850 100644 --- a/benchmark/run_speed_bench_50.py +++ b/latency_benchmarks/run_speed_bench_50.py @@ -23,8 +23,8 @@ import psutil # Ensure project root is in sys.path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from pipeline.orchestrator import get_orchestrator -from pipeline.schemas import QueryRequest, QueryResponse +from rag_pipeline.orchestrator import get_orchestrator +from rag_pipeline.schemas import QueryRequest, QueryResponse logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("speed_bench_50") diff --git a/generation/answer_cache.py b/llm_synthesis/answer_cache.py similarity index 100% rename from generation/answer_cache.py rename to llm_synthesis/answer_cache.py diff --git a/generation/extractive.py b/llm_synthesis/extractive.py similarity index 98% rename from generation/extractive.py rename to llm_synthesis/extractive.py index 1e7681bd83c736de3651fb933b6199ddcb92b9bc..cc102d33eff0b3f8ac3208e7576f3ac1edc59bcf 100644 --- a/generation/extractive.py +++ b/llm_synthesis/extractive.py @@ -9,8 +9,8 @@ from typing import Any, Dict, List, Optional import numpy as np import re import config -from chunking.metadata import split_sentences_multilingual -from generation.answer_cache import get_answer_cache +from doc_chunking.metadata import split_sentences_multilingual +from llm_synthesis.answer_cache import get_answer_cache def extract_answer_from_passage( diff --git a/generation/llm_fallback.py b/llm_synthesis/llm_fallback.py similarity index 100% rename from generation/llm_fallback.py rename to llm_synthesis/llm_fallback.py diff --git a/generation/local_slm.py b/llm_synthesis/local_slm.py similarity index 100% rename from generation/local_slm.py rename to llm_synthesis/local_slm.py diff --git a/training/prepare_rag_sft_data.py b/model_training/prepare_rag_sft_data.py similarity index 100% rename from training/prepare_rag_sft_data.py rename to model_training/prepare_rag_sft_data.py diff --git a/training/train_colab.py b/model_training/train_colab.py similarity index 99% rename from training/train_colab.py rename to model_training/train_colab.py index 3200e1939609c79f78cb6238dfad6289c6e1f3a6..fe95156875cf38b35a4c29adf4f7cb47f9f26657 100644 --- a/training/train_colab.py +++ b/model_training/train_colab.py @@ -12,7 +12,7 @@ Usage in Google Colab: import os import json import torch -from datasets import load_dataset +from knowledge_base import load_dataset from transformers import ( AutoModelForCausalLM, AutoTokenizer, diff --git a/training/train_qwen_rag_colab.ipynb b/model_training/train_qwen_rag_colab.ipynb similarity index 100% rename from training/train_qwen_rag_colab.ipynb rename to model_training/train_qwen_rag_colab.ipynb diff --git a/training/train_qwen_unsloth_colab.ipynb b/model_training/train_qwen_unsloth_colab.ipynb similarity index 100% rename from training/train_qwen_unsloth_colab.ipynb rename to model_training/train_qwen_unsloth_colab.ipynb diff --git a/training/upload_to_adaption.py b/model_training/upload_to_adaption.py similarity index 100% rename from training/upload_to_adaption.py rename to model_training/upload_to_adaption.py diff --git a/pipeline/orchestrator.py b/rag_pipeline/orchestrator.py similarity index 97% rename from pipeline/orchestrator.py rename to rag_pipeline/orchestrator.py index 1cc143b1489a5c45be4277a6b251f56f59aa9dcb..29c00f07fa3faff121cb56975496cc15084a68ca 100644 --- a/pipeline/orchestrator.py +++ b/rag_pipeline/orchestrator.py @@ -17,24 +17,24 @@ from typing import Any, Dict, List, Optional import numpy as np import config -from pipeline.schemas import ( +from rag_pipeline.schemas import ( GuardrailFlags, QueryRequest, QueryResponse, RetrievedChunk, StageTiming, ) -from retrieval.embed import get_embedder -from retrieval.index_faiss import get_index_manager -from retrieval.rerank import rerank_bm25_hybrid, rerank_cross_encoder, get_cross_encoder -from chunking.hybrid_merge import merge_and_fuse_candidates -from guardrails.pre_retrieval import check_unsafe_content, check_off_topic_query, check_query_intent, get_safety_telemetry -from guardrails.prompt_guard import get_prompt_guard_detector -from guardrails.post_generation import check_grounding, DECLINED_RESPONSE_TEMPLATE -from generation.extractive import generate_extractive -from generation.answer_cache import get_answer_cache -from generation.llm_fallback import get_llm_adapter -from stt.sarvam_client import get_stt_client +from vector_search.embed import get_embedder +from vector_search.index_faiss import get_index_manager +from vector_search.rerank import rerank_bm25_hybrid, rerank_cross_encoder, get_cross_encoder +from doc_chunking.hybrid_merge import merge_and_fuse_candidates +from safety_guardrails.pre_retrieval import check_unsafe_content, check_off_topic_query, check_query_intent, get_safety_telemetry +from safety_guardrails.prompt_guard import get_prompt_guard_detector +from safety_guardrails.post_generation import check_grounding, DECLINED_RESPONSE_TEMPLATE +from llm_synthesis.extractive import generate_extractive +from llm_synthesis.answer_cache import get_answer_cache +from llm_synthesis.llm_fallback import get_llm_adapter +from voice_stt.sarvam_client import get_stt_client logger = logging.getLogger(__name__) @@ -550,7 +550,7 @@ class RAGPipelineOrchestrator: if config.ENABLE_LOCAL_SLM: # High-speed local offline SLM generation on CPU (< 60 ms) - from generation.local_slm import get_local_slm_adapter + from llm_synthesis.local_slm import get_local_slm_adapter context_blocks = [] for i, c in enumerate(reranked_chunks[:3]): lang_code = c.get("source_lang", "UNK").upper() diff --git a/pipeline/schemas.py b/rag_pipeline/schemas.py similarity index 100% rename from pipeline/schemas.py rename to rag_pipeline/schemas.py diff --git a/app/__init__.py b/rag_service/__init__.py similarity index 100% rename from app/__init__.py rename to rag_service/__init__.py diff --git a/app/benchmark.py b/rag_service/benchmark.py similarity index 95% rename from app/benchmark.py rename to rag_service/benchmark.py index 59d1659f59a8985eede8a53b90ad06d3b9c2ba73..8bbfa9976affaa68dfb4446f095efec61b99f25a 100644 --- a/app/benchmark.py +++ b/rag_service/benchmark.py @@ -13,8 +13,8 @@ _ROOT_DIR = Path(__file__).resolve().parent.parent if str(_ROOT_DIR) not in sys.path: sys.path.insert(0, str(_ROOT_DIR)) -from app.config import LATENCY_BUDGET_MS -from app.retriever import search, warmup +from rag_service.config import LATENCY_BUDGET_MS +from rag_service.retriever import search, warmup QUERIES = [ "What is FAISS used for?", diff --git a/app/config.py b/rag_service/config.py similarity index 100% rename from app/config.py rename to rag_service/config.py diff --git a/app/retriever.py b/rag_service/retriever.py similarity index 96% rename from app/retriever.py rename to rag_service/retriever.py index 9fe69a0b6ad02e39267484a296f4263c18616679..5776fc526c1ec0efc9b9ecef32f9cb0045da3f97 100644 --- a/app/retriever.py +++ b/rag_service/retriever.py @@ -18,8 +18,8 @@ if str(_ROOT_DIR) not in sys.path: sys.path.insert(0, str(_ROOT_DIR)) import config -from retrieval.embed import get_embedder -from retrieval.index_faiss import get_index_manager +from vector_search.embed import get_embedder +from vector_search.index_faiss import get_index_manager logger = logging.getLogger(__name__) diff --git a/guardrails/fail_safe.py b/safety_guardrails/fail_safe.py similarity index 100% rename from guardrails/fail_safe.py rename to safety_guardrails/fail_safe.py diff --git a/guardrails/patterns_ext.py b/safety_guardrails/patterns_ext.py similarity index 100% rename from guardrails/patterns_ext.py rename to safety_guardrails/patterns_ext.py diff --git a/guardrails/post_generation.py b/safety_guardrails/post_generation.py similarity index 100% rename from guardrails/post_generation.py rename to safety_guardrails/post_generation.py diff --git a/guardrails/pre_retrieval.py b/safety_guardrails/pre_retrieval.py similarity index 99% rename from guardrails/pre_retrieval.py rename to safety_guardrails/pre_retrieval.py index 3d1a7b889476a495c700ace283e62e85c34b9dfc..bbea0268802f510a8d7ce59079b69ad2d851d6f4 100644 --- a/guardrails/pre_retrieval.py +++ b/safety_guardrails/pre_retrieval.py @@ -16,9 +16,9 @@ import urllib.error from typing import Any, Dict, List, Optional, Tuple import numpy as np import config -from retrieval.embed import get_embedder -from guardrails.prompt_guard import get_prompt_guard_detector, PromptGuardResult -from guardrails.patterns_ext import UNSAFE_PATTERN_EXTENSIONS, INTENT_PATTERN_EXTENSIONS +from vector_search.embed import get_embedder +from safety_guardrails.prompt_guard import get_prompt_guard_detector, PromptGuardResult +from safety_guardrails.patterns_ext import UNSAFE_PATTERN_EXTENSIONS, INTENT_PATTERN_EXTENSIONS logger = logging.getLogger(__name__) diff --git a/guardrails/prompt_guard.py b/safety_guardrails/prompt_guard.py similarity index 99% rename from guardrails/prompt_guard.py rename to safety_guardrails/prompt_guard.py index 343b271d363ad7748926c46616e79da43ed578ea..158e12fb9e5403302b701c76a932a7d68edf1df8 100644 --- a/guardrails/prompt_guard.py +++ b/safety_guardrails/prompt_guard.py @@ -24,7 +24,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import config -from guardrails.fail_safe import evaluate_fail_safe +from safety_guardrails.fail_safe import evaluate_fail_safe logger = logging.getLogger(__name__) diff --git a/tests/test_eval_fixes.py b/tests/test_eval_fixes.py index 58d40d9dc8a52afcccd690fa012921346ccd4abd..0c27500b602952a8ad95feda55051e704fa5b791 100644 --- a/tests/test_eval_fixes.py +++ b/tests/test_eval_fixes.py @@ -21,15 +21,15 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from guardrails.pre_retrieval import ( +from safety_guardrails.pre_retrieval import ( check_unsafe_content, check_query_intent, check_off_topic_query, get_safety_telemetry, ) -from guardrails.prompt_guard import get_prompt_guard_detector, PromptGuardResult, PromptGuardDetector -from pipeline.schemas import QueryRequest, QueryResponse, GuardrailFlags -from pipeline.orchestrator import get_orchestrator, RAGPipelineOrchestrator +from safety_guardrails.prompt_guard import get_prompt_guard_detector, PromptGuardResult, PromptGuardDetector +from rag_pipeline.schemas import QueryRequest, QueryResponse, GuardrailFlags +from rag_pipeline.orchestrator import get_orchestrator, RAGPipelineOrchestrator class TestDiagnosisFailedExamples: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a9a370c7ab1376c8fe951aa04c520b30f8868465..e12d308c2b9efb7eabf0aca4b1461c848ebbc44e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -13,24 +13,24 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from chunking.metadata import ( +from doc_chunking.metadata import ( Chunk, filter_chunks_by_language, split_sentences_multilingual, calculate_overlap_tokens, estimate_token_count, ) -from chunking.passage_native import chunk_passage_native -from chunking.sentence_window import chunk_document_sentence_window -from chunking.semantic import chunk_document_semantic -from chunking.hybrid_merge import merge_and_fuse_candidates -from retrieval.rerank import rerank_bm25_hybrid, tokenize_for_bm25 -from guardrails.pre_retrieval import check_unsafe_content, check_off_topic_query -from guardrails.post_generation import check_grounding, compute_lexical_grounding_score -from generation.extractive import extract_answer_from_passage, generate_extractive -from generation.llm_fallback import LLMAdapter -from pipeline.schemas import QueryRequest, QueryResponse, StageTiming -from pipeline.orchestrator import get_orchestrator +from doc_chunking.passage_native import chunk_passage_native +from doc_chunking.sentence_window import chunk_document_sentence_window +from doc_chunking.semantic import chunk_document_semantic +from doc_chunking.hybrid_merge import merge_and_fuse_candidates +from vector_search.rerank import rerank_bm25_hybrid, tokenize_for_bm25 +from safety_guardrails.pre_retrieval import check_unsafe_content, check_off_topic_query +from safety_guardrails.post_generation import check_grounding, compute_lexical_grounding_score +from llm_synthesis.extractive import extract_answer_from_passage, generate_extractive +from llm_synthesis.llm_fallback import LLMAdapter +from rag_pipeline.schemas import QueryRequest, QueryResponse, StageTiming +from rag_pipeline.orchestrator import get_orchestrator class TestLanguageExtensibility: @@ -296,7 +296,7 @@ class TestEndToEndPipeline: assert len(resp.answer) > 20 def test_robust_json_parser_handles_markdown_and_edge_cases(self): - from guardrails.pre_retrieval import robust_json_parser + from safety_guardrails.pre_retrieval import robust_json_parser # Standard JSON p1 = robust_json_parser('{"is_safe": true, "reason": "ok"}') diff --git a/tests/test_prompt_guard.py b/tests/test_prompt_guard.py index 609e23a56f19f2dc2e2dcda660c7e8a39a9e1cc8..151192b2be275d63f6987f0db8dc083518bfd131 100644 --- a/tests/test_prompt_guard.py +++ b/tests/test_prompt_guard.py @@ -4,8 +4,8 @@ Unit and benchmark tests for Meta Prompt-Guard 86M Sub-10ms Guardrail. import time import pytest -from guardrails.prompt_guard import get_prompt_guard_detector, PromptGuardResult -from guardrails.pre_retrieval import normalize_and_unpack_text, check_unsafe_content +from safety_guardrails.prompt_guard import get_prompt_guard_detector, PromptGuardResult +from safety_guardrails.pre_retrieval import normalize_and_unpack_text, check_unsafe_content @pytest.fixture(scope="module") diff --git a/retrieval/build_indexes_gpu.py b/vector_search/build_indexes_gpu.py similarity index 97% rename from retrieval/build_indexes_gpu.py rename to vector_search/build_indexes_gpu.py index 8623abb79a9426409d764e46972f0413956a540b..b2472a986f023f7e27c444cf1ba2bbb805e80ce1 100644 --- a/retrieval/build_indexes_gpu.py +++ b/vector_search/build_indexes_gpu.py @@ -19,11 +19,11 @@ import torch # Ensure project root in sys.path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from chunking.metadata import Chunk -from chunking.passage_native import process_corpus_passage_native -from chunking.sentence_window import process_longdocs_sentence_window -from chunking.semantic import process_longdocs_semantic -from retrieval.index_faiss import StrategyVectorIndex +from doc_chunking.metadata import Chunk +from doc_chunking.passage_native import process_corpus_passage_native +from doc_chunking.sentence_window import process_longdocs_sentence_window +from doc_chunking.semantic import process_longdocs_semantic +from vector_search.index_faiss import StrategyVectorIndex logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", force=True) logger = logging.getLogger(__name__) diff --git a/retrieval/embed.py b/vector_search/embed.py similarity index 100% rename from retrieval/embed.py rename to vector_search/embed.py diff --git a/retrieval/index_all_languages_gpu.py b/vector_search/index_all_languages_gpu.py similarity index 97% rename from retrieval/index_all_languages_gpu.py rename to vector_search/index_all_languages_gpu.py index 02a153a3e0e519470dd02ed8da7b53f6a06feaf4..650a1927679830a681dc209146500e059bdac4f5 100644 --- a/retrieval/index_all_languages_gpu.py +++ b/vector_search/index_all_languages_gpu.py @@ -21,11 +21,11 @@ import torch # Ensure project root in sys.path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from chunking.metadata import Chunk -from chunking.passage_native import process_corpus_passage_native -from chunking.sentence_window import process_longdocs_sentence_window -from chunking.semantic import process_longdocs_semantic -from retrieval.index_faiss import StrategyVectorIndex +from doc_chunking.metadata import Chunk +from doc_chunking.passage_native import process_corpus_passage_native +from doc_chunking.sentence_window import process_longdocs_sentence_window +from doc_chunking.semantic import process_longdocs_semantic +from vector_search.index_faiss import StrategyVectorIndex logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", force=True) logger = logging.getLogger(__name__) diff --git a/retrieval/index_faiss.py b/vector_search/index_faiss.py similarity index 98% rename from retrieval/index_faiss.py rename to vector_search/index_faiss.py index c3bbd656d8c166db93bfe37fff4ded62d8ccf156..cf1db534e6c2f457c73cf7435fabc07b3e79153e 100644 --- a/retrieval/index_faiss.py +++ b/vector_search/index_faiss.py @@ -21,11 +21,11 @@ import numpy as np # Ensure project root is in sys.path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from chunking.metadata import Chunk, filter_chunks_by_language -from chunking.passage_native import process_corpus_passage_native -from chunking.sentence_window import process_longdocs_sentence_window -from chunking.semantic import process_longdocs_semantic -from retrieval.embed import get_embedder +from doc_chunking.metadata import Chunk, filter_chunks_by_language +from doc_chunking.passage_native import process_corpus_passage_native +from doc_chunking.sentence_window import process_longdocs_sentence_window +from doc_chunking.semantic import process_longdocs_semantic +from vector_search.embed import get_embedder logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", force=True) logger = logging.getLogger(__name__) diff --git a/retrieval/rerank.py b/vector_search/rerank.py similarity index 100% rename from retrieval/rerank.py rename to vector_search/rerank.py diff --git a/stt/sarvam_client.py b/voice_stt/sarvam_client.py similarity index 100% rename from stt/sarvam_client.py rename to voice_stt/sarvam_client.py diff --git a/demo/bg_beach.png b/web_ui/bg_beach.png similarity index 100% rename from demo/bg_beach.png rename to web_ui/bg_beach.png diff --git a/demo/cap.png b/web_ui/cap.png similarity index 100% rename from demo/cap.png rename to web_ui/cap.png diff --git a/demo/cli_demo.py b/web_ui/cli_demo.py similarity index 97% rename from demo/cli_demo.py rename to web_ui/cli_demo.py index b4900d9e111820abefd478b827dc3fab3d4ac109..2533399da0ccc6f2676b23df3b642ee488e0195a 100644 --- a/demo/cli_demo.py +++ b/web_ui/cli_demo.py @@ -27,8 +27,8 @@ except Exception: # Ensure project root is in sys.path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import config -from pipeline.orchestrator import get_orchestrator -from pipeline.schemas import QueryRequest, QueryResponse +from rag_pipeline.orchestrator import get_orchestrator +from rag_pipeline.schemas import QueryRequest, QueryResponse logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("cli_demo") diff --git a/demo/index.html b/web_ui/index.html similarity index 100% rename from demo/index.html rename to web_ui/index.html diff --git a/demo/ironman.png b/web_ui/ironman.png similarity index 100% rename from demo/ironman.png rename to web_ui/ironman.png diff --git a/demo/logo.svg b/web_ui/logo.svg similarity index 100% rename from demo/logo.svg rename to web_ui/logo.svg diff --git a/web_ui/spider_gwen.png b/web_ui/spider_gwen.png new file mode 100644 index 0000000000000000000000000000000000000000..f143c8c0b38a3b52bbde78deef559644d397fc6c --- /dev/null +++ b/web_ui/spider_gwen.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a4889b94aa90b8e623b0fbcbff50b8f4177c25b46589adff471c3085318601d +size 159117 diff --git a/web_ui/spider_gwen_clean.png b/web_ui/spider_gwen_clean.png new file mode 100644 index 0000000000000000000000000000000000000000..8b54303738de4b12f4caf79718057288fca275af --- /dev/null +++ b/web_ui/spider_gwen_clean.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8de068367541b116f2fcef66b49cbb36f0fb39baa0f96850e212f2ea0ee0d25 +size 97631 diff --git a/web_ui/spider_gwen_cutout.png b/web_ui/spider_gwen_cutout.png new file mode 100644 index 0000000000000000000000000000000000000000..11a2ee125c3f6a6e5466567971c13cde654df8fd --- /dev/null +++ b/web_ui/spider_gwen_cutout.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69897055ed2eb22d3bba9c096441fd42bbadf8b91fadc0296916fa4cd07fc5af +size 128740 diff --git a/demo/thor.png b/web_ui/thor.png similarity index 100% rename from demo/thor.png rename to web_ui/thor.png