Remove Nemotron integration
Browse filesCo-authored-by: Codex <codex@openai.com>
- README.md +0 -11
- config.py +2 -23
- modal_nemotron_service.py +0 -165
- services/providers.py +1 -8
- services/rag_pipeline.py +21 -221
- tests/test_rag_pipeline.py +1 -82
README.md
CHANGED
|
@@ -54,8 +54,6 @@ The hackathon-safe default is Qwen3 14B through the Hugging Face router.
|
|
| 54 |
- Default credential path: `HF_TOKEN`
|
| 55 |
- Default provider: `qwen`
|
| 56 |
|
| 57 |
-
Nemotron support remains optional for experimentation through the alternate provider path, but it is not the default app configuration.
|
| 58 |
-
|
| 59 |
## Local Run
|
| 60 |
|
| 61 |
```bash
|
|
@@ -67,15 +65,6 @@ Set these environment variables for the default Qwen path:
|
|
| 67 |
|
| 68 |
- `HF_TOKEN`
|
| 69 |
|
| 70 |
-
Optional Nemotron environment variables are still supported for non-default experimentation:
|
| 71 |
-
|
| 72 |
-
- `NEMOTRON_API_KEY`
|
| 73 |
-
- `NEMOTRON_BASE_URL`
|
| 74 |
-
- `NEMOTRON_RETRIEVER_API_KEY`
|
| 75 |
-
- `NEMOTRON_RETRIEVER_BASE_URL`
|
| 76 |
-
- `NEMOTRON_RERANKER_API_KEY`
|
| 77 |
-
- `NEMOTRON_RERANKER_URL`
|
| 78 |
-
|
| 79 |
## Space Deployment
|
| 80 |
|
| 81 |
This directory is structured as a Hugging Face Space:
|
|
|
|
| 54 |
- Default credential path: `HF_TOKEN`
|
| 55 |
- Default provider: `qwen`
|
| 56 |
|
|
|
|
|
|
|
| 57 |
## Local Run
|
| 58 |
|
| 59 |
```bash
|
|
|
|
| 65 |
|
| 66 |
- `HF_TOKEN`
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
## Space Deployment
|
| 69 |
|
| 70 |
This directory is structured as a Hugging Face Space:
|
config.py
CHANGED
|
@@ -32,14 +32,11 @@ warnings.filterwarnings(
|
|
| 32 |
category=FutureWarning,
|
| 33 |
)
|
| 34 |
|
| 35 |
-
SUPPORTED_PROVIDERS = ["
|
| 36 |
DEFAULT_PROVIDER: str = "qwen"
|
| 37 |
-
DEFAULT_NEMOTRON_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
|
| 38 |
DEFAULT_QWEN_MODEL = "Qwen/Qwen3-14B:cheapest"
|
| 39 |
DEFAULT_CHUNK_TOKENIZER_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
| 40 |
DEFAULT_FALLBACK_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
| 41 |
-
DEFAULT_NEMOTRON_RETRIEVER_MODEL = "nvidia/llama-nemotron-embed-vl-1b-v2"
|
| 42 |
-
DEFAULT_NEMOTRON_RERANKER_MODEL = "nvidia/llama-nemotron-rerank-1b-v2"
|
| 43 |
OPENAI_REASONING_EFFORT = "medium"
|
| 44 |
ANTHROPIC_THINKING_BUDGET = 2048
|
| 45 |
DEFAULT_CHUNK_SIZE = 350
|
|
@@ -53,12 +50,11 @@ TOP_K_RETRIEVAL = 5
|
|
| 53 |
MAX_UPLOAD_SIZE_MB = 25
|
| 54 |
TIMEOUT_SECONDS = 30
|
| 55 |
|
| 56 |
-
ProviderLiteral = Literal["
|
| 57 |
|
| 58 |
# Conservative full-document QA input budgets derived from provider/model
|
| 59 |
# context-window docs, with headroom reserved for prompts and outputs.
|
| 60 |
PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS: dict[ProviderLiteral, int] = {
|
| 61 |
-
"nemotron": 64_000,
|
| 62 |
"qwen": 24_000,
|
| 63 |
"openai": 900_000,
|
| 64 |
"anthropic": 900_000,
|
|
@@ -89,26 +85,9 @@ GEMINI_API_KEY: Optional[str] = _read_env_key("GEMINI_API_KEY")
|
|
| 89 |
COHERE_API_KEY: Optional[str] = _read_env_key("COHERE_API_KEY")
|
| 90 |
DEFAULT_COHERE_KEY: Optional[str] = _read_env_key("DEFAULT_COHERE_KEY")
|
| 91 |
HF_TOKEN: Optional[str] = _read_env_key("HF_TOKEN")
|
| 92 |
-
NEMOTRON_API_KEY: Optional[str] = _read_env_key("NEMOTRON_API_KEY") or HF_TOKEN
|
| 93 |
-
NEMOTRON_BASE_URL: str = os.getenv("NEMOTRON_BASE_URL", "https://router.huggingface.co/v1").strip() or "https://router.huggingface.co/v1"
|
| 94 |
-
NEMOTRON_RETRIEVER_API_KEY: Optional[str] = _read_env_key("NEMOTRON_RETRIEVER_API_KEY") or NEMOTRON_API_KEY
|
| 95 |
-
NEMOTRON_RETRIEVER_BASE_URL: str = (
|
| 96 |
-
os.getenv("NEMOTRON_RETRIEVER_BASE_URL", NEMOTRON_BASE_URL).strip() or NEMOTRON_BASE_URL
|
| 97 |
-
)
|
| 98 |
-
NEMOTRON_RERANKER_API_KEY: Optional[str] = _read_env_key("NEMOTRON_RERANKER_API_KEY") or NEMOTRON_API_KEY
|
| 99 |
-
NEMOTRON_RERANKER_URL: Optional[str] = _read_env_key("NEMOTRON_RERANKER_URL")
|
| 100 |
|
| 101 |
|
| 102 |
PROVIDER_METADATA: list[ProviderConfig] = [
|
| 103 |
-
ProviderConfig(
|
| 104 |
-
name="nemotron",
|
| 105 |
-
key_prefix=None,
|
| 106 |
-
display_name="Nemotron Nano 30B",
|
| 107 |
-
instructions=(
|
| 108 |
-
"Use NEMOTRON_API_KEY for a self-hosted or external OpenAI-compatible Nemotron endpoint. "
|
| 109 |
-
"This is optional and no longer the default app path."
|
| 110 |
-
),
|
| 111 |
-
),
|
| 112 |
ProviderConfig(
|
| 113 |
name="qwen",
|
| 114 |
key_prefix=None,
|
|
|
|
| 32 |
category=FutureWarning,
|
| 33 |
)
|
| 34 |
|
| 35 |
+
SUPPORTED_PROVIDERS = ["qwen", "openai", "anthropic", "gemini", "cohere"]
|
| 36 |
DEFAULT_PROVIDER: str = "qwen"
|
|
|
|
| 37 |
DEFAULT_QWEN_MODEL = "Qwen/Qwen3-14B:cheapest"
|
| 38 |
DEFAULT_CHUNK_TOKENIZER_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
| 39 |
DEFAULT_FALLBACK_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
|
|
|
|
|
|
| 40 |
OPENAI_REASONING_EFFORT = "medium"
|
| 41 |
ANTHROPIC_THINKING_BUDGET = 2048
|
| 42 |
DEFAULT_CHUNK_SIZE = 350
|
|
|
|
| 50 |
MAX_UPLOAD_SIZE_MB = 25
|
| 51 |
TIMEOUT_SECONDS = 30
|
| 52 |
|
| 53 |
+
ProviderLiteral = Literal["qwen", "openai", "anthropic", "gemini", "cohere"]
|
| 54 |
|
| 55 |
# Conservative full-document QA input budgets derived from provider/model
|
| 56 |
# context-window docs, with headroom reserved for prompts and outputs.
|
| 57 |
PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS: dict[ProviderLiteral, int] = {
|
|
|
|
| 58 |
"qwen": 24_000,
|
| 59 |
"openai": 900_000,
|
| 60 |
"anthropic": 900_000,
|
|
|
|
| 85 |
COHERE_API_KEY: Optional[str] = _read_env_key("COHERE_API_KEY")
|
| 86 |
DEFAULT_COHERE_KEY: Optional[str] = _read_env_key("DEFAULT_COHERE_KEY")
|
| 87 |
HF_TOKEN: Optional[str] = _read_env_key("HF_TOKEN")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
|
| 90 |
PROVIDER_METADATA: list[ProviderConfig] = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
ProviderConfig(
|
| 92 |
name="qwen",
|
| 93 |
key_prefix=None,
|
modal_nemotron_service.py
DELETED
|
@@ -1,165 +0,0 @@
|
|
| 1 |
-
"""Modal deployment for an OpenAI-compatible Nemotron generator endpoint.
|
| 2 |
-
|
| 3 |
-
This service solves the current Hugging Face router gap for
|
| 4 |
-
`nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16` by serving the model directly on
|
| 5 |
-
Modal with SGLang. The app can then point `NEMOTRON_BASE_URL` at the deployed
|
| 6 |
-
Modal URL and continue using the existing OpenAI client path.
|
| 7 |
-
"""
|
| 8 |
-
|
| 9 |
-
from __future__ import annotations
|
| 10 |
-
|
| 11 |
-
import os
|
| 12 |
-
import subprocess
|
| 13 |
-
import time
|
| 14 |
-
|
| 15 |
-
import modal
|
| 16 |
-
import modal.experimental
|
| 17 |
-
|
| 18 |
-
APP_NAME = os.getenv("MODAL_APP_NAME", "legislation-explainer-nemotron")
|
| 19 |
-
MODEL_NAME = os.getenv(
|
| 20 |
-
"NEMOTRON_MODAL_MODEL",
|
| 21 |
-
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
|
| 22 |
-
)
|
| 23 |
-
PORT = int(os.getenv("NEMOTRON_MODAL_PORT", "8000"))
|
| 24 |
-
GPU = os.getenv("NEMOTRON_MODAL_GPU", "H100")
|
| 25 |
-
REGION = os.getenv("NEMOTRON_MODAL_REGION", "us-east")
|
| 26 |
-
PROXY_REGION = os.getenv("NEMOTRON_MODAL_PROXY_REGION", "us-east")
|
| 27 |
-
MIN_CONTAINERS = int(os.getenv("NEMOTRON_MODAL_MIN_CONTAINERS", "0"))
|
| 28 |
-
SCALEDOWN_WINDOW = int(os.getenv("NEMOTRON_MODAL_SCALEDOWN_WINDOW", "300"))
|
| 29 |
-
TIMEOUT_SECONDS = int(os.getenv("NEMOTRON_MODAL_TIMEOUT_SECONDS", "3600"))
|
| 30 |
-
STARTUP_TIMEOUT_SECONDS = int(os.getenv("NEMOTRON_MODAL_STARTUP_TIMEOUT_SECONDS", "3600"))
|
| 31 |
-
TARGET_INPUTS = int(os.getenv("NEMOTRON_MODAL_TARGET_INPUTS", "8"))
|
| 32 |
-
HF_SECRET_NAME = os.getenv("NEMOTRON_MODAL_HF_SECRET_NAME", "huggingface-secret")
|
| 33 |
-
VOLUME_NAME = os.getenv("NEMOTRON_MODAL_VOLUME_NAME", "legislation-explainer-nemotron-cache")
|
| 34 |
-
HF_CACHE_PATH = "/root/.cache/huggingface"
|
| 35 |
-
|
| 36 |
-
server_image = (
|
| 37 |
-
modal.Image.from_registry("lmsysorg/sglang:v0.5.11")
|
| 38 |
-
.entrypoint([])
|
| 39 |
-
.run_commands("rm -rf /root/.cache/huggingface")
|
| 40 |
-
.env(
|
| 41 |
-
{
|
| 42 |
-
"HF_HUB_CACHE": HF_CACHE_PATH,
|
| 43 |
-
"HF_XET_HIGH_PERFORMANCE": "1",
|
| 44 |
-
"SAFETENSORS_FAST_GPU": "1",
|
| 45 |
-
"NVIDIA_TF32_OVERRIDE": "1",
|
| 46 |
-
"SGLANG_ENABLE_SPEC_V2": "1",
|
| 47 |
-
"SGLANG_ENABLE_JIT_DEEPGEMM": "0",
|
| 48 |
-
}
|
| 49 |
-
)
|
| 50 |
-
)
|
| 51 |
-
|
| 52 |
-
with server_image.imports():
|
| 53 |
-
import requests
|
| 54 |
-
|
| 55 |
-
app = modal.App(APP_NAME, image=server_image)
|
| 56 |
-
hf_secret = modal.Secret.from_name(HF_SECRET_NAME)
|
| 57 |
-
hf_cache_volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True)
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def _check_running(process: subprocess.Popen[str]) -> None:
|
| 61 |
-
return_code = process.poll()
|
| 62 |
-
if return_code is not None:
|
| 63 |
-
raise subprocess.CalledProcessError(return_code, cmd=process.args)
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
def _wait_ready(process: subprocess.Popen[str], timeout_seconds: int = 1800) -> None:
|
| 67 |
-
deadline = time.time() + timeout_seconds
|
| 68 |
-
while time.time() < deadline:
|
| 69 |
-
try:
|
| 70 |
-
_check_running(process)
|
| 71 |
-
response = requests.get(f"http://127.0.0.1:{PORT}/health", timeout=5)
|
| 72 |
-
response.raise_for_status()
|
| 73 |
-
return
|
| 74 |
-
except (
|
| 75 |
-
requests.exceptions.ConnectionError,
|
| 76 |
-
requests.exceptions.HTTPError,
|
| 77 |
-
requests.exceptions.Timeout,
|
| 78 |
-
subprocess.CalledProcessError,
|
| 79 |
-
):
|
| 80 |
-
time.sleep(5)
|
| 81 |
-
raise TimeoutError(f"SGLang server was not ready within {timeout_seconds} seconds.")
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
def _warm_up_server() -> None:
|
| 85 |
-
payload = {
|
| 86 |
-
"model": MODEL_NAME,
|
| 87 |
-
"messages": [{"role": "user", "content": "Reply with the single word READY."}],
|
| 88 |
-
"max_tokens": 8,
|
| 89 |
-
"temperature": 0,
|
| 90 |
-
}
|
| 91 |
-
for _ in range(2):
|
| 92 |
-
response = requests.post(
|
| 93 |
-
f"http://127.0.0.1:{PORT}/v1/chat/completions",
|
| 94 |
-
json=payload,
|
| 95 |
-
timeout=30,
|
| 96 |
-
)
|
| 97 |
-
response.raise_for_status()
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
@app.cls(
|
| 101 |
-
gpu=GPU,
|
| 102 |
-
min_containers=MIN_CONTAINERS,
|
| 103 |
-
scaledown_window=SCALEDOWN_WINDOW,
|
| 104 |
-
timeout=TIMEOUT_SECONDS,
|
| 105 |
-
startup_timeout=STARTUP_TIMEOUT_SECONDS,
|
| 106 |
-
region=REGION,
|
| 107 |
-
secrets=[hf_secret],
|
| 108 |
-
volumes={HF_CACHE_PATH: hf_cache_volume},
|
| 109 |
-
)
|
| 110 |
-
@modal.experimental.http_server(
|
| 111 |
-
port=PORT,
|
| 112 |
-
proxy_regions=[PROXY_REGION],
|
| 113 |
-
exit_grace_period=15,
|
| 114 |
-
)
|
| 115 |
-
@modal.concurrent(target_inputs=TARGET_INPUTS)
|
| 116 |
-
class NemotronGenerator:
|
| 117 |
-
"""Serve Nemotron through SGLang's OpenAI-compatible HTTP API."""
|
| 118 |
-
|
| 119 |
-
process: subprocess.Popen[str] | None = None
|
| 120 |
-
|
| 121 |
-
@modal.enter()
|
| 122 |
-
def start_server(self) -> None:
|
| 123 |
-
command = [
|
| 124 |
-
"sglang",
|
| 125 |
-
"serve",
|
| 126 |
-
"--model-path",
|
| 127 |
-
MODEL_NAME,
|
| 128 |
-
"--served-model-name",
|
| 129 |
-
MODEL_NAME,
|
| 130 |
-
"--host",
|
| 131 |
-
"0.0.0.0",
|
| 132 |
-
"--port",
|
| 133 |
-
str(PORT),
|
| 134 |
-
"--trust-remote-code",
|
| 135 |
-
"--reasoning-parser",
|
| 136 |
-
"nemotron_3",
|
| 137 |
-
"--context-length",
|
| 138 |
-
os.getenv("NEMOTRON_MODAL_CONTEXT_LENGTH", "32768"),
|
| 139 |
-
"--chunked-prefill-size",
|
| 140 |
-
os.getenv("NEMOTRON_MODAL_CHUNKED_PREFILL_SIZE", "8192"),
|
| 141 |
-
"--mem-fraction-static",
|
| 142 |
-
os.getenv("NEMOTRON_MODAL_MEM_FRACTION_STATIC", "0.82"),
|
| 143 |
-
]
|
| 144 |
-
self.process = subprocess.Popen(command)
|
| 145 |
-
_wait_ready(self.process)
|
| 146 |
-
_warm_up_server()
|
| 147 |
-
|
| 148 |
-
@modal.exit()
|
| 149 |
-
def stop_server(self) -> None:
|
| 150 |
-
if self.process is None:
|
| 151 |
-
return
|
| 152 |
-
if self.process.poll() is None:
|
| 153 |
-
self.process.terminate()
|
| 154 |
-
try:
|
| 155 |
-
self.process.wait(timeout=20)
|
| 156 |
-
except subprocess.TimeoutExpired:
|
| 157 |
-
self.process.kill()
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
@app.local_entrypoint()
|
| 161 |
-
def main() -> None:
|
| 162 |
-
print(
|
| 163 |
-
"Deploy with `modal deploy modal_nemotron_service.py`. "
|
| 164 |
-
"Then set NEMOTRON_BASE_URL to the deployed service URL plus `/v1`."
|
| 165 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
services/providers.py
CHANGED
|
@@ -13,12 +13,9 @@ from config import (
|
|
| 13 |
COHERE_API_KEY,
|
| 14 |
DEFAULT_PROVIDER,
|
| 15 |
DEFAULT_COHERE_KEY,
|
| 16 |
-
DEFAULT_NEMOTRON_MODEL,
|
| 17 |
GEMINI_API_KEY,
|
| 18 |
DEFAULT_QWEN_MODEL,
|
| 19 |
HF_TOKEN,
|
| 20 |
-
NEMOTRON_API_KEY,
|
| 21 |
-
NEMOTRON_BASE_URL,
|
| 22 |
OPENAI_API_KEY,
|
| 23 |
PROVIDER_METADATA,
|
| 24 |
ProviderConfig,
|
|
@@ -37,7 +34,6 @@ class ProviderClient:
|
|
| 37 |
|
| 38 |
|
| 39 |
_PROVIDER_MODEL_MAP: Dict[ProviderLiteral, str] = {
|
| 40 |
-
"nemotron": DEFAULT_NEMOTRON_MODEL,
|
| 41 |
"qwen": DEFAULT_QWEN_MODEL,
|
| 42 |
"openai": "gpt-5.5",
|
| 43 |
"anthropic": "claude-sonnet-4-6",
|
|
@@ -47,7 +43,6 @@ _PROVIDER_MODEL_MAP: Dict[ProviderLiteral, str] = {
|
|
| 47 |
|
| 48 |
|
| 49 |
_DEFAULT_PROVIDER_KEYS: Dict[ProviderLiteral, Optional[str]] = {
|
| 50 |
-
"nemotron": NEMOTRON_API_KEY or HF_TOKEN,
|
| 51 |
"qwen": HF_TOKEN,
|
| 52 |
"openai": OPENAI_API_KEY,
|
| 53 |
"anthropic": ANTHROPIC_API_KEY,
|
|
@@ -84,9 +79,7 @@ def validate_api_key(provider: ProviderLiteral, api_key: str | None) -> tuple[bo
|
|
| 84 |
|
| 85 |
|
| 86 |
def instantiate_client(provider: ProviderLiteral, api_key: str) -> ProviderClient:
|
| 87 |
-
if provider == "
|
| 88 |
-
client = OpenAI(api_key=api_key, base_url=NEMOTRON_BASE_URL)
|
| 89 |
-
elif provider == "qwen":
|
| 90 |
client = OpenAI(api_key=api_key, base_url="https://router.huggingface.co/v1")
|
| 91 |
elif provider == "openai":
|
| 92 |
client = OpenAI(api_key=api_key)
|
|
|
|
| 13 |
COHERE_API_KEY,
|
| 14 |
DEFAULT_PROVIDER,
|
| 15 |
DEFAULT_COHERE_KEY,
|
|
|
|
| 16 |
GEMINI_API_KEY,
|
| 17 |
DEFAULT_QWEN_MODEL,
|
| 18 |
HF_TOKEN,
|
|
|
|
|
|
|
| 19 |
OPENAI_API_KEY,
|
| 20 |
PROVIDER_METADATA,
|
| 21 |
ProviderConfig,
|
|
|
|
| 34 |
|
| 35 |
|
| 36 |
_PROVIDER_MODEL_MAP: Dict[ProviderLiteral, str] = {
|
|
|
|
| 37 |
"qwen": DEFAULT_QWEN_MODEL,
|
| 38 |
"openai": "gpt-5.5",
|
| 39 |
"anthropic": "claude-sonnet-4-6",
|
|
|
|
| 43 |
|
| 44 |
|
| 45 |
_DEFAULT_PROVIDER_KEYS: Dict[ProviderLiteral, Optional[str]] = {
|
|
|
|
| 46 |
"qwen": HF_TOKEN,
|
| 47 |
"openai": OPENAI_API_KEY,
|
| 48 |
"anthropic": ANTHROPIC_API_KEY,
|
|
|
|
| 79 |
|
| 80 |
|
| 81 |
def instantiate_client(provider: ProviderLiteral, api_key: str) -> ProviderClient:
|
| 82 |
+
if provider == "qwen":
|
|
|
|
|
|
|
| 83 |
client = OpenAI(api_key=api_key, base_url="https://router.huggingface.co/v1")
|
| 84 |
elif provider == "openai":
|
| 85 |
client = OpenAI(api_key=api_key)
|
services/rag_pipeline.py
CHANGED
|
@@ -5,7 +5,6 @@ from __future__ import annotations
|
|
| 5 |
import hashlib
|
| 6 |
import inspect
|
| 7 |
import json
|
| 8 |
-
import math
|
| 9 |
import re
|
| 10 |
from concurrent.futures import ThreadPoolExecutor
|
| 11 |
from dataclasses import dataclass, field
|
|
@@ -33,9 +32,7 @@ from langchain_community.embeddings import SentenceTransformerEmbeddings
|
|
| 33 |
from langchain_community.vectorstores import FAISS
|
| 34 |
from langchain_core.documents import Document
|
| 35 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 36 |
-
from openai import BadRequestError, OpenAI
|
| 37 |
from pydantic import BaseModel, Field
|
| 38 |
-
import httpx
|
| 39 |
from transformers import AutoTokenizer
|
| 40 |
|
| 41 |
from config import (
|
|
@@ -44,13 +41,6 @@ from config import (
|
|
| 44 |
DEFAULT_CHUNK_SIZE,
|
| 45 |
DEFAULT_CHUNK_TOKENIZER_MODEL,
|
| 46 |
DEFAULT_FALLBACK_EMBEDDING_MODEL,
|
| 47 |
-
DEFAULT_QWEN_MODEL,
|
| 48 |
-
DEFAULT_NEMOTRON_RETRIEVER_MODEL,
|
| 49 |
-
DEFAULT_NEMOTRON_RERANKER_MODEL,
|
| 50 |
-
NEMOTRON_RETRIEVER_API_KEY,
|
| 51 |
-
NEMOTRON_RETRIEVER_BASE_URL,
|
| 52 |
-
NEMOTRON_RERANKER_API_KEY,
|
| 53 |
-
NEMOTRON_RERANKER_URL,
|
| 54 |
OPENAI_REASONING_EFFORT,
|
| 55 |
PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS,
|
| 56 |
SCAN_BATCH_SIZE,
|
|
@@ -58,7 +48,6 @@ from config import (
|
|
| 58 |
SCAN_CHUNK_SIZE,
|
| 59 |
SCAN_MAX_WINDOWS,
|
| 60 |
SCAN_TOP_K,
|
| 61 |
-
TIMEOUT_SECONDS,
|
| 62 |
TOP_K_RETRIEVAL,
|
| 63 |
)
|
| 64 |
from services.precomputed_assets import PrecomputedExampleAsset, load_precomputed_asset_for_url
|
|
@@ -139,14 +128,6 @@ class AnalysisSnippet:
|
|
| 139 |
section: str
|
| 140 |
text: str
|
| 141 |
|
| 142 |
-
|
| 143 |
-
@dataclass
|
| 144 |
-
class NemotronRetrievalIndex:
|
| 145 |
-
chunks: list[str]
|
| 146 |
-
embeddings: list[list[float]]
|
| 147 |
-
model_name: str = DEFAULT_NEMOTRON_RETRIEVER_MODEL
|
| 148 |
-
|
| 149 |
-
|
| 150 |
@dataclass
|
| 151 |
class CachedDocumentArtifacts:
|
| 152 |
document_hash: str
|
|
@@ -162,7 +143,7 @@ class CachedDocumentArtifacts:
|
|
| 162 |
|
| 163 |
|
| 164 |
_DOCUMENT_CACHE: dict[str, CachedDocumentArtifacts] = {}
|
| 165 |
-
RetrievalIndex = FAISS
|
| 166 |
|
| 167 |
|
| 168 |
@lru_cache(maxsize=1)
|
|
@@ -177,8 +158,7 @@ def _embedding_tokenizer(model_name: str = DEFAULT_CHUNK_TOKENIZER_MODEL) -> Tok
|
|
| 177 |
|
| 178 |
def warm_embedding_stack(model_name: str = DEFAULT_CHUNK_TOKENIZER_MODEL) -> None:
|
| 179 |
_embedding_tokenizer(model_name)
|
| 180 |
-
|
| 181 |
-
_fallback_embedding_model()
|
| 182 |
|
| 183 |
|
| 184 |
def _split_with_tokenizer(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
|
|
@@ -233,60 +213,6 @@ def _anthropic_thinking_kwargs(provider_client: ProviderClient) -> dict[str, Any
|
|
| 233 |
}
|
| 234 |
|
| 235 |
|
| 236 |
-
def _fallback_generation_model(provider_client: ProviderClient) -> str | None:
|
| 237 |
-
if provider_client.name == "nemotron":
|
| 238 |
-
return DEFAULT_QWEN_MODEL
|
| 239 |
-
return None
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
def _is_model_not_supported_error(exc: BadRequestError) -> bool:
|
| 243 |
-
body = getattr(exc, "body", None)
|
| 244 |
-
if isinstance(body, dict):
|
| 245 |
-
error_block = body.get("error")
|
| 246 |
-
if isinstance(error_block, dict):
|
| 247 |
-
code = str(error_block.get("code", "")).strip().lower()
|
| 248 |
-
message = str(error_block.get("message", "")).strip().lower()
|
| 249 |
-
else:
|
| 250 |
-
code = str(body.get("code", "")).strip().lower()
|
| 251 |
-
message = str(body.get("message", "")).strip().lower()
|
| 252 |
-
if code == "model_not_supported":
|
| 253 |
-
return True
|
| 254 |
-
if "not supported by any provider" in message:
|
| 255 |
-
return True
|
| 256 |
-
return False
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
def _chat_completion_with_fallback(
|
| 260 |
-
provider_client: ProviderClient,
|
| 261 |
-
*,
|
| 262 |
-
messages: list[dict[str, str]],
|
| 263 |
-
temperature: float = 0,
|
| 264 |
-
) -> Any:
|
| 265 |
-
candidate_models = [provider_client.default_model]
|
| 266 |
-
fallback_model = _fallback_generation_model(provider_client)
|
| 267 |
-
if fallback_model and fallback_model not in candidate_models:
|
| 268 |
-
candidate_models.append(fallback_model)
|
| 269 |
-
|
| 270 |
-
last_error: Exception | None = None
|
| 271 |
-
for index, model_name in enumerate(candidate_models):
|
| 272 |
-
try:
|
| 273 |
-
return provider_client.client.chat.completions.create(
|
| 274 |
-
model=model_name,
|
| 275 |
-
messages=messages,
|
| 276 |
-
temperature=temperature,
|
| 277 |
-
)
|
| 278 |
-
except BadRequestError as exc:
|
| 279 |
-
last_error = exc
|
| 280 |
-
is_last_model = index == len(candidate_models) - 1
|
| 281 |
-
if not is_last_model and _is_model_not_supported_error(exc):
|
| 282 |
-
continue
|
| 283 |
-
raise
|
| 284 |
-
|
| 285 |
-
if last_error is not None:
|
| 286 |
-
raise last_error
|
| 287 |
-
raise RuntimeError("No candidate generation models were available.")
|
| 288 |
-
|
| 289 |
-
|
| 290 |
def split_into_chunks(text: str, chunk_size: int = DEFAULT_CHUNK_SIZE, chunk_overlap: int = DEFAULT_CHUNK_OVERLAP) -> List[str]:
|
| 291 |
return _split_with_tokenizer(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
| 292 |
|
|
@@ -306,39 +232,8 @@ def build_vector_store(chunks: Iterable[str]) -> FAISS:
|
|
| 306 |
return FAISS.from_documents(docs, embedding=embeddings)
|
| 307 |
|
| 308 |
|
| 309 |
-
def _nemotron_retrieval_ready() -> bool:
|
| 310 |
-
return bool(NEMOTRON_RETRIEVER_BASE_URL and NEMOTRON_RETRIEVER_API_KEY and DEFAULT_NEMOTRON_RETRIEVER_MODEL)
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
def _nemotron_reranker_ready() -> bool:
|
| 314 |
-
return bool(NEMOTRON_RERANKER_URL and NEMOTRON_RERANKER_API_KEY and DEFAULT_NEMOTRON_RERANKER_MODEL)
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
def _embed_texts_with_nemotron(texts: list[str]) -> list[list[float]]:
|
| 318 |
-
if not texts:
|
| 319 |
-
return []
|
| 320 |
-
if not _nemotron_retrieval_ready():
|
| 321 |
-
raise RuntimeError("Nemotron retriever endpoint is not configured.")
|
| 322 |
-
|
| 323 |
-
client = OpenAI(api_key=NEMOTRON_RETRIEVER_API_KEY, base_url=NEMOTRON_RETRIEVER_BASE_URL)
|
| 324 |
-
response = client.embeddings.create(model=DEFAULT_NEMOTRON_RETRIEVER_MODEL, input=texts)
|
| 325 |
-
ordered = sorted(response.data, key=lambda item: item.index)
|
| 326 |
-
return [list(item.embedding) for item in ordered]
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
def build_nemotron_retrieval_index(chunks: Iterable[str]) -> NemotronRetrievalIndex:
|
| 330 |
-
chunk_list = list(chunks)
|
| 331 |
-
return NemotronRetrievalIndex(chunks=chunk_list, embeddings=_embed_texts_with_nemotron(chunk_list))
|
| 332 |
-
|
| 333 |
-
|
| 334 |
def build_retrieval_index(chunks: Iterable[str]) -> RetrievalIndex:
|
| 335 |
-
|
| 336 |
-
if _nemotron_retrieval_ready():
|
| 337 |
-
try:
|
| 338 |
-
return build_nemotron_retrieval_index(chunk_list)
|
| 339 |
-
except Exception:
|
| 340 |
-
pass
|
| 341 |
-
return build_vector_store(chunk_list)
|
| 342 |
|
| 343 |
|
| 344 |
def document_hash_for_text(text: str) -> str:
|
|
@@ -346,15 +241,7 @@ def document_hash_for_text(text: str) -> str:
|
|
| 346 |
|
| 347 |
|
| 348 |
def _retrieval_signature() -> str:
|
| 349 |
-
return "|".join(
|
| 350 |
-
[
|
| 351 |
-
DEFAULT_CHUNK_TOKENIZER_MODEL,
|
| 352 |
-
DEFAULT_FALLBACK_EMBEDDING_MODEL,
|
| 353 |
-
DEFAULT_NEMOTRON_RETRIEVER_MODEL,
|
| 354 |
-
DEFAULT_NEMOTRON_RERANKER_MODEL if _nemotron_reranker_ready() else "no-rerank",
|
| 355 |
-
"nemotron" if _nemotron_retrieval_ready() else "fallback",
|
| 356 |
-
]
|
| 357 |
-
)
|
| 358 |
|
| 359 |
|
| 360 |
def document_cache_key(document_hash: str) -> str:
|
|
@@ -544,9 +431,9 @@ def _generate_json_payload(
|
|
| 544 |
**_openai_reasoning_kwargs(provider_client),
|
| 545 |
)
|
| 546 |
return _extract_openai_json(response)
|
| 547 |
-
if provider_client.name
|
| 548 |
-
response =
|
| 549 |
-
provider_client,
|
| 550 |
messages=[
|
| 551 |
{"role": "system", "content": prompt},
|
| 552 |
{
|
|
@@ -966,9 +853,9 @@ def _answer_question_from_full_document(
|
|
| 966 |
**_openai_reasoning_kwargs(provider_client),
|
| 967 |
)
|
| 968 |
return _extract_openai_text(response)
|
| 969 |
-
if provider_client.name
|
| 970 |
-
response =
|
| 971 |
-
provider_client,
|
| 972 |
messages=[
|
| 973 |
{"role": "system", "content": prompt},
|
| 974 |
{"role": "user", "content": user_text},
|
|
@@ -1008,75 +895,14 @@ def _answer_question_from_full_document(
|
|
| 1008 |
return _extract_gemini_text(response)
|
| 1009 |
|
| 1010 |
|
| 1011 |
-
def _cosine_similarity(left: list[float], right: list[float]) -> float:
|
| 1012 |
-
dot_product = sum(a * b for a, b in zip(left, right))
|
| 1013 |
-
left_norm = math.sqrt(sum(value * value for value in left))
|
| 1014 |
-
right_norm = math.sqrt(sum(value * value for value in right))
|
| 1015 |
-
if left_norm == 0 or right_norm == 0:
|
| 1016 |
-
return 0.0
|
| 1017 |
-
return dot_product / (left_norm * right_norm)
|
| 1018 |
-
|
| 1019 |
-
|
| 1020 |
-
def _rerank_candidates(question: str, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
| 1021 |
-
if not candidates or not _nemotron_reranker_ready():
|
| 1022 |
-
return candidates
|
| 1023 |
-
|
| 1024 |
-
headers = {"Content-Type": "application/json"}
|
| 1025 |
-
if NEMOTRON_RERANKER_API_KEY:
|
| 1026 |
-
headers["Authorization"] = f"Bearer {NEMOTRON_RERANKER_API_KEY}"
|
| 1027 |
-
|
| 1028 |
-
payload = {
|
| 1029 |
-
"model": DEFAULT_NEMOTRON_RERANKER_MODEL,
|
| 1030 |
-
"query": question,
|
| 1031 |
-
"documents": [candidate["text"] for candidate in candidates],
|
| 1032 |
-
"top_n": len(candidates),
|
| 1033 |
-
}
|
| 1034 |
-
try:
|
| 1035 |
-
response = httpx.post(NEMOTRON_RERANKER_URL, json=payload, headers=headers, timeout=TIMEOUT_SECONDS)
|
| 1036 |
-
response.raise_for_status()
|
| 1037 |
-
body = response.json()
|
| 1038 |
-
except Exception:
|
| 1039 |
-
return candidates
|
| 1040 |
-
|
| 1041 |
-
results = body.get("results") or body.get("data") or []
|
| 1042 |
-
if not isinstance(results, list):
|
| 1043 |
-
return candidates
|
| 1044 |
-
|
| 1045 |
-
ranked_candidates: list[tuple[float, int, dict[str, Any]]] = []
|
| 1046 |
-
for item in results:
|
| 1047 |
-
if not isinstance(item, dict):
|
| 1048 |
-
continue
|
| 1049 |
-
index = item.get("index")
|
| 1050 |
-
if not isinstance(index, int) or index < 0 or index >= len(candidates):
|
| 1051 |
-
continue
|
| 1052 |
-
score = item.get("relevance_score", item.get("score", 0.0))
|
| 1053 |
-
try:
|
| 1054 |
-
numeric_score = float(score)
|
| 1055 |
-
except (TypeError, ValueError):
|
| 1056 |
-
numeric_score = 0.0
|
| 1057 |
-
ranked_candidates.append((numeric_score, index, candidates[index]))
|
| 1058 |
-
|
| 1059 |
-
if not ranked_candidates:
|
| 1060 |
-
return candidates
|
| 1061 |
-
ranked_candidates.sort(key=lambda item: (-item[0], item[1]))
|
| 1062 |
-
return [candidate for _, _, candidate in ranked_candidates]
|
| 1063 |
-
|
| 1064 |
-
|
| 1065 |
def _retrieve_context_from_faiss(vector_store: FAISS, question: str) -> tuple[str, list[Citation]]:
|
| 1066 |
-
retrieved_docs = vector_store.similarity_search(question, k=max(TOP_K_RETRIEVAL * 2, TOP_K_RETRIEVAL))
|
| 1067 |
-
candidates = [
|
| 1068 |
-
{
|
| 1069 |
-
"chunk_id": idx,
|
| 1070 |
-
"text": doc.page_content.strip(),
|
| 1071 |
-
}
|
| 1072 |
-
for idx, doc in enumerate(retrieved_docs, start=1)
|
| 1073 |
-
if doc.page_content.strip()
|
| 1074 |
-
]
|
| 1075 |
-
ranked_candidates = _rerank_candidates(question, candidates)[:TOP_K_RETRIEVAL]
|
| 1076 |
context_blocks: list[str] = []
|
| 1077 |
citations: list[Citation] = []
|
| 1078 |
-
|
| 1079 |
-
|
|
|
|
|
|
|
|
|
|
| 1080 |
context_blocks.append(f"[{idx}] {snippet}")
|
| 1081 |
citations.append(Citation(ref_id=idx, snippet=snippet))
|
| 1082 |
|
|
@@ -1084,33 +910,7 @@ def _retrieve_context_from_faiss(vector_store: FAISS, question: str) -> tuple[st
|
|
| 1084 |
return context_text, citations
|
| 1085 |
|
| 1086 |
|
| 1087 |
-
def _retrieve_context_from_nemotron_index(vector_store: NemotronRetrievalIndex, question: str) -> tuple[str, list[Citation]]:
|
| 1088 |
-
query_embedding = _embed_texts_with_nemotron([question])[0]
|
| 1089 |
-
scored_candidates = [
|
| 1090 |
-
{
|
| 1091 |
-
"chunk_id": idx,
|
| 1092 |
-
"text": chunk.strip(),
|
| 1093 |
-
"score": _cosine_similarity(query_embedding, embedding),
|
| 1094 |
-
}
|
| 1095 |
-
for idx, (chunk, embedding) in enumerate(zip(vector_store.chunks, vector_store.embeddings), start=1)
|
| 1096 |
-
if chunk.strip()
|
| 1097 |
-
]
|
| 1098 |
-
scored_candidates.sort(key=lambda item: (-item["score"], item["chunk_id"]))
|
| 1099 |
-
ranked_candidates = _rerank_candidates(question, scored_candidates[: max(TOP_K_RETRIEVAL * 3, TOP_K_RETRIEVAL)])[:TOP_K_RETRIEVAL]
|
| 1100 |
-
|
| 1101 |
-
context_blocks: list[str] = []
|
| 1102 |
-
citations: list[Citation] = []
|
| 1103 |
-
for ref_id, candidate in enumerate(ranked_candidates, start=1):
|
| 1104 |
-
snippet = candidate["text"]
|
| 1105 |
-
citations.append(Citation(ref_id=ref_id, snippet=snippet))
|
| 1106 |
-
context_blocks.append(f"[{ref_id}] {snippet}")
|
| 1107 |
-
context_text = "\n\n".join(context_blocks) if context_blocks else "No context available."
|
| 1108 |
-
return context_text, citations
|
| 1109 |
-
|
| 1110 |
-
|
| 1111 |
def _retrieve_context_from_index(vector_store: RetrievalIndex, question: str) -> tuple[str, list[Citation]]:
|
| 1112 |
-
if isinstance(vector_store, NemotronRetrievalIndex):
|
| 1113 |
-
return _retrieve_context_from_nemotron_index(vector_store, question)
|
| 1114 |
return _retrieve_context_from_faiss(vector_store, question)
|
| 1115 |
|
| 1116 |
|
|
@@ -1137,9 +937,9 @@ def _generate_answer_from_context(provider_client: ProviderClient, context_text:
|
|
| 1137 |
**_openai_reasoning_kwargs(provider_client),
|
| 1138 |
)
|
| 1139 |
answer = _extract_openai_text(response)
|
| 1140 |
-
elif provider_client.name
|
| 1141 |
-
response =
|
| 1142 |
-
provider_client,
|
| 1143 |
messages=[
|
| 1144 |
{"role": "system", "content": prompt},
|
| 1145 |
{
|
|
@@ -1310,9 +1110,9 @@ def _scan_batch(provider_client: ProviderClient, batch: list[ScanChunk], questio
|
|
| 1310 |
**_openai_reasoning_kwargs(provider_client),
|
| 1311 |
)
|
| 1312 |
return _extract_openai_json(response)
|
| 1313 |
-
if provider_client.name
|
| 1314 |
-
response =
|
| 1315 |
-
provider_client,
|
| 1316 |
messages=[
|
| 1317 |
{"role": "system", "content": prompt},
|
| 1318 |
{"role": "user", "content": user_text},
|
|
|
|
| 5 |
import hashlib
|
| 6 |
import inspect
|
| 7 |
import json
|
|
|
|
| 8 |
import re
|
| 9 |
from concurrent.futures import ThreadPoolExecutor
|
| 10 |
from dataclasses import dataclass, field
|
|
|
|
| 32 |
from langchain_community.vectorstores import FAISS
|
| 33 |
from langchain_core.documents import Document
|
| 34 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
|
| 35 |
from pydantic import BaseModel, Field
|
|
|
|
| 36 |
from transformers import AutoTokenizer
|
| 37 |
|
| 38 |
from config import (
|
|
|
|
| 41 |
DEFAULT_CHUNK_SIZE,
|
| 42 |
DEFAULT_CHUNK_TOKENIZER_MODEL,
|
| 43 |
DEFAULT_FALLBACK_EMBEDDING_MODEL,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
OPENAI_REASONING_EFFORT,
|
| 45 |
PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS,
|
| 46 |
SCAN_BATCH_SIZE,
|
|
|
|
| 48 |
SCAN_CHUNK_SIZE,
|
| 49 |
SCAN_MAX_WINDOWS,
|
| 50 |
SCAN_TOP_K,
|
|
|
|
| 51 |
TOP_K_RETRIEVAL,
|
| 52 |
)
|
| 53 |
from services.precomputed_assets import PrecomputedExampleAsset, load_precomputed_asset_for_url
|
|
|
|
| 128 |
section: str
|
| 129 |
text: str
|
| 130 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
@dataclass
|
| 132 |
class CachedDocumentArtifacts:
|
| 133 |
document_hash: str
|
|
|
|
| 143 |
|
| 144 |
|
| 145 |
_DOCUMENT_CACHE: dict[str, CachedDocumentArtifacts] = {}
|
| 146 |
+
RetrievalIndex = FAISS
|
| 147 |
|
| 148 |
|
| 149 |
@lru_cache(maxsize=1)
|
|
|
|
| 158 |
|
| 159 |
def warm_embedding_stack(model_name: str = DEFAULT_CHUNK_TOKENIZER_MODEL) -> None:
|
| 160 |
_embedding_tokenizer(model_name)
|
| 161 |
+
_fallback_embedding_model()
|
|
|
|
| 162 |
|
| 163 |
|
| 164 |
def _split_with_tokenizer(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
|
|
|
|
| 213 |
}
|
| 214 |
|
| 215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
def split_into_chunks(text: str, chunk_size: int = DEFAULT_CHUNK_SIZE, chunk_overlap: int = DEFAULT_CHUNK_OVERLAP) -> List[str]:
|
| 217 |
return _split_with_tokenizer(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
| 218 |
|
|
|
|
| 232 |
return FAISS.from_documents(docs, embedding=embeddings)
|
| 233 |
|
| 234 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
def build_retrieval_index(chunks: Iterable[str]) -> RetrievalIndex:
|
| 236 |
+
return build_vector_store(list(chunks))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
|
| 238 |
|
| 239 |
def document_hash_for_text(text: str) -> str:
|
|
|
|
| 241 |
|
| 242 |
|
| 243 |
def _retrieval_signature() -> str:
|
| 244 |
+
return "|".join([DEFAULT_CHUNK_TOKENIZER_MODEL, DEFAULT_FALLBACK_EMBEDDING_MODEL, "faiss"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
|
| 246 |
|
| 247 |
def document_cache_key(document_hash: str) -> str:
|
|
|
|
| 431 |
**_openai_reasoning_kwargs(provider_client),
|
| 432 |
)
|
| 433 |
return _extract_openai_json(response)
|
| 434 |
+
if provider_client.name == "qwen":
|
| 435 |
+
response = provider_client.client.chat.completions.create(
|
| 436 |
+
model=provider_client.default_model,
|
| 437 |
messages=[
|
| 438 |
{"role": "system", "content": prompt},
|
| 439 |
{
|
|
|
|
| 853 |
**_openai_reasoning_kwargs(provider_client),
|
| 854 |
)
|
| 855 |
return _extract_openai_text(response)
|
| 856 |
+
if provider_client.name == "qwen":
|
| 857 |
+
response = provider_client.client.chat.completions.create(
|
| 858 |
+
model=provider_client.default_model,
|
| 859 |
messages=[
|
| 860 |
{"role": "system", "content": prompt},
|
| 861 |
{"role": "user", "content": user_text},
|
|
|
|
| 895 |
return _extract_gemini_text(response)
|
| 896 |
|
| 897 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
def _retrieve_context_from_faiss(vector_store: FAISS, question: str) -> tuple[str, list[Citation]]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 899 |
context_blocks: list[str] = []
|
| 900 |
citations: list[Citation] = []
|
| 901 |
+
retrieved_docs = vector_store.similarity_search(question, k=TOP_K_RETRIEVAL)
|
| 902 |
+
for idx, doc in enumerate(retrieved_docs, start=1):
|
| 903 |
+
snippet = doc.page_content.strip()
|
| 904 |
+
if not snippet:
|
| 905 |
+
continue
|
| 906 |
context_blocks.append(f"[{idx}] {snippet}")
|
| 907 |
citations.append(Citation(ref_id=idx, snippet=snippet))
|
| 908 |
|
|
|
|
| 910 |
return context_text, citations
|
| 911 |
|
| 912 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 913 |
def _retrieve_context_from_index(vector_store: RetrievalIndex, question: str) -> tuple[str, list[Citation]]:
|
|
|
|
|
|
|
| 914 |
return _retrieve_context_from_faiss(vector_store, question)
|
| 915 |
|
| 916 |
|
|
|
|
| 937 |
**_openai_reasoning_kwargs(provider_client),
|
| 938 |
)
|
| 939 |
answer = _extract_openai_text(response)
|
| 940 |
+
elif provider_client.name == "qwen":
|
| 941 |
+
response = provider_client.client.chat.completions.create(
|
| 942 |
+
model=provider_client.default_model,
|
| 943 |
messages=[
|
| 944 |
{"role": "system", "content": prompt},
|
| 945 |
{
|
|
|
|
| 1110 |
**_openai_reasoning_kwargs(provider_client),
|
| 1111 |
)
|
| 1112 |
return _extract_openai_json(response)
|
| 1113 |
+
if provider_client.name == "qwen":
|
| 1114 |
+
response = provider_client.client.chat.completions.create(
|
| 1115 |
+
model=provider_client.default_model,
|
| 1116 |
messages=[
|
| 1117 |
{"role": "system", "content": prompt},
|
| 1118 |
{"role": "user", "content": user_text},
|
tests/test_rag_pipeline.py
CHANGED
|
@@ -1,21 +1,16 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import json
|
| 4 |
-
|
| 5 |
-
from openai import BadRequestError
|
| 6 |
-
|
| 7 |
-
from config import DEFAULT_NEMOTRON_MODEL, DEFAULT_QWEN_MODEL, OPENAI_REASONING_EFFORT
|
| 8 |
from services.providers import ProviderClient
|
| 9 |
from services.providers import _PROVIDER_MODEL_MAP
|
| 10 |
from services.rag_pipeline import (
|
| 11 |
AnswerResult,
|
| 12 |
AnalysisResult,
|
| 13 |
Citation,
|
| 14 |
-
NemotronRetrievalIndex,
|
| 15 |
ScanMatch,
|
| 16 |
_anthropic_thinking_kwargs,
|
| 17 |
_analysis_answer_schema,
|
| 18 |
-
_chat_completion_with_fallback,
|
| 19 |
_full_document_budget,
|
| 20 |
_analysis_schema,
|
| 21 |
_extract_chat_completion_text,
|
|
@@ -224,7 +219,6 @@ def test_gemini_default_model_is_thinking_capable():
|
|
| 224 |
|
| 225 |
|
| 226 |
def test_provider_defaults_use_thinking_capable_models():
|
| 227 |
-
assert _PROVIDER_MODEL_MAP["nemotron"] == DEFAULT_NEMOTRON_MODEL
|
| 228 |
assert _PROVIDER_MODEL_MAP["openai"] == "gpt-5.5"
|
| 229 |
assert _PROVIDER_MODEL_MAP["anthropic"] == "claude-sonnet-4-6"
|
| 230 |
assert _PROVIDER_MODEL_MAP["cohere"] == "command-a-reasoning-08-2025"
|
|
@@ -238,53 +232,6 @@ def test_extract_chat_completion_text_strips_qwen_thinking_block():
|
|
| 238 |
assert _extract_chat_completion_text(response) == "Final answer"
|
| 239 |
|
| 240 |
|
| 241 |
-
def test_chat_completion_with_fallback_uses_qwen_when_nemotron_is_unsupported():
|
| 242 |
-
unsupported = BadRequestError(
|
| 243 |
-
message="unsupported",
|
| 244 |
-
response=type(
|
| 245 |
-
"Response",
|
| 246 |
-
(),
|
| 247 |
-
{
|
| 248 |
-
"request": None,
|
| 249 |
-
"status_code": 400,
|
| 250 |
-
"headers": {},
|
| 251 |
-
"json": lambda self: {
|
| 252 |
-
"error": {
|
| 253 |
-
"message": "The requested model is not supported by any provider you have enabled.",
|
| 254 |
-
"code": "model_not_supported",
|
| 255 |
-
}
|
| 256 |
-
},
|
| 257 |
-
"text": "",
|
| 258 |
-
},
|
| 259 |
-
)(),
|
| 260 |
-
body={
|
| 261 |
-
"error": {
|
| 262 |
-
"message": "The requested model is not supported by any provider you have enabled.",
|
| 263 |
-
"code": "model_not_supported",
|
| 264 |
-
}
|
| 265 |
-
},
|
| 266 |
-
)
|
| 267 |
-
message = type("Message", (), {"content": "Fallback answer"})()
|
| 268 |
-
choice = type("Choice", (), {"message": message})()
|
| 269 |
-
success = type("Response", (), {"choices": [choice]})()
|
| 270 |
-
client = DummyOpenAIChatClient([unsupported, success])
|
| 271 |
-
provider_client = ProviderClient(
|
| 272 |
-
name="nemotron",
|
| 273 |
-
client=client,
|
| 274 |
-
default_model=DEFAULT_NEMOTRON_MODEL,
|
| 275 |
-
api_key="dummy-api-key",
|
| 276 |
-
)
|
| 277 |
-
|
| 278 |
-
response = _chat_completion_with_fallback(
|
| 279 |
-
provider_client,
|
| 280 |
-
messages=[{"role": "user", "content": "Question"}],
|
| 281 |
-
temperature=0,
|
| 282 |
-
)
|
| 283 |
-
|
| 284 |
-
assert client.chat.completions.calls == [DEFAULT_NEMOTRON_MODEL, DEFAULT_QWEN_MODEL]
|
| 285 |
-
assert response.choices[0].message.content == "Fallback answer"
|
| 286 |
-
|
| 287 |
-
|
| 288 |
def test_answer_query_uses_scan_context(monkeypatch):
|
| 289 |
provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
|
| 290 |
analysis = AnalysisResult(executive_summary="Creates a new authority.")
|
|
@@ -402,34 +349,6 @@ def test_answer_query_from_full_document_uses_scan_when_full_document_too_large(
|
|
| 402 |
assert result.citations[0].snippet == "licensing rule"
|
| 403 |
|
| 404 |
|
| 405 |
-
def test_retrieve_context_from_nemotron_index_uses_reranked_candidates(monkeypatch):
|
| 406 |
-
index = NemotronRetrievalIndex(
|
| 407 |
-
chunks=[
|
| 408 |
-
"Clause 1 sets startup licensing fees at a fixed annual amount.",
|
| 409 |
-
"Clause 7 defines the minister's emergency powers.",
|
| 410 |
-
"Clause 11 outlines reporting deadlines for data fiduciaries.",
|
| 411 |
-
],
|
| 412 |
-
embeddings=[
|
| 413 |
-
[0.2, 0.1, 0.9],
|
| 414 |
-
[0.8, 0.1, 0.1],
|
| 415 |
-
[0.1, 0.9, 0.2],
|
| 416 |
-
],
|
| 417 |
-
)
|
| 418 |
-
|
| 419 |
-
monkeypatch.setattr(
|
| 420 |
-
"services.rag_pipeline._embed_texts_with_nemotron",
|
| 421 |
-
lambda texts: [[0.0, 0.0, 1.0]],
|
| 422 |
-
)
|
| 423 |
-
monkeypatch.setattr(
|
| 424 |
-
"services.rag_pipeline._rerank_candidates",
|
| 425 |
-
lambda question, candidates: [candidates[2], candidates[0], candidates[1]],
|
| 426 |
-
)
|
| 427 |
-
|
| 428 |
-
context_text, citations = _retrieve_context_from_index(index, "What emergency powers does the minister have?")
|
| 429 |
-
assert citations[0].snippet == "Clause 7 defines the minister's emergency powers."
|
| 430 |
-
assert "[1] Clause 7 defines the minister's emergency powers." in context_text
|
| 431 |
-
|
| 432 |
-
|
| 433 |
def test_answer_query_requests_deeper_answer_when_analysis_is_insufficient(monkeypatch):
|
| 434 |
provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
|
| 435 |
analysis = AnalysisResult(executive_summary="A short summary.")
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import json
|
| 4 |
+
from config import DEFAULT_QWEN_MODEL, OPENAI_REASONING_EFFORT
|
|
|
|
|
|
|
|
|
|
| 5 |
from services.providers import ProviderClient
|
| 6 |
from services.providers import _PROVIDER_MODEL_MAP
|
| 7 |
from services.rag_pipeline import (
|
| 8 |
AnswerResult,
|
| 9 |
AnalysisResult,
|
| 10 |
Citation,
|
|
|
|
| 11 |
ScanMatch,
|
| 12 |
_anthropic_thinking_kwargs,
|
| 13 |
_analysis_answer_schema,
|
|
|
|
| 14 |
_full_document_budget,
|
| 15 |
_analysis_schema,
|
| 16 |
_extract_chat_completion_text,
|
|
|
|
| 219 |
|
| 220 |
|
| 221 |
def test_provider_defaults_use_thinking_capable_models():
|
|
|
|
| 222 |
assert _PROVIDER_MODEL_MAP["openai"] == "gpt-5.5"
|
| 223 |
assert _PROVIDER_MODEL_MAP["anthropic"] == "claude-sonnet-4-6"
|
| 224 |
assert _PROVIDER_MODEL_MAP["cohere"] == "command-a-reasoning-08-2025"
|
|
|
|
| 232 |
assert _extract_chat_completion_text(response) == "Final answer"
|
| 233 |
|
| 234 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
def test_answer_query_uses_scan_context(monkeypatch):
|
| 236 |
provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
|
| 237 |
analysis = AnalysisResult(executive_summary="Creates a new authority.")
|
|
|
|
| 349 |
assert result.citations[0].snippet == "licensing rule"
|
| 350 |
|
| 351 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
def test_answer_query_requests_deeper_answer_when_analysis_is_insufficient(monkeypatch):
|
| 353 |
provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
|
| 354 |
analysis = AnalysisResult(executive_summary="A short summary.")
|