KayO Codex commited on
Commit
482bf38
·
1 Parent(s): 99bb596

Remove Nemotron integration

Browse files

Co-authored-by: Codex <codex@openai.com>

README.md CHANGED
@@ -54,8 +54,6 @@ The hackathon-safe default is Qwen3 14B through the Hugging Face router.
54
  - Default credential path: `HF_TOKEN`
55
  - Default provider: `qwen`
56
 
57
- Nemotron support remains optional for experimentation through the alternate provider path, but it is not the default app configuration.
58
-
59
  ## Local Run
60
 
61
  ```bash
@@ -67,15 +65,6 @@ Set these environment variables for the default Qwen path:
67
 
68
  - `HF_TOKEN`
69
 
70
- Optional Nemotron environment variables are still supported for non-default experimentation:
71
-
72
- - `NEMOTRON_API_KEY`
73
- - `NEMOTRON_BASE_URL`
74
- - `NEMOTRON_RETRIEVER_API_KEY`
75
- - `NEMOTRON_RETRIEVER_BASE_URL`
76
- - `NEMOTRON_RERANKER_API_KEY`
77
- - `NEMOTRON_RERANKER_URL`
78
-
79
  ## Space Deployment
80
 
81
  This directory is structured as a Hugging Face Space:
 
54
  - Default credential path: `HF_TOKEN`
55
  - Default provider: `qwen`
56
 
 
 
57
  ## Local Run
58
 
59
  ```bash
 
65
 
66
  - `HF_TOKEN`
67
 
 
 
 
 
 
 
 
 
 
68
  ## Space Deployment
69
 
70
  This directory is structured as a Hugging Face Space:
config.py CHANGED
@@ -32,14 +32,11 @@ warnings.filterwarnings(
32
  category=FutureWarning,
33
  )
34
 
35
- SUPPORTED_PROVIDERS = ["nemotron", "qwen", "openai", "anthropic", "gemini", "cohere"]
36
  DEFAULT_PROVIDER: str = "qwen"
37
- DEFAULT_NEMOTRON_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
38
  DEFAULT_QWEN_MODEL = "Qwen/Qwen3-14B:cheapest"
39
  DEFAULT_CHUNK_TOKENIZER_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
40
  DEFAULT_FALLBACK_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
41
- DEFAULT_NEMOTRON_RETRIEVER_MODEL = "nvidia/llama-nemotron-embed-vl-1b-v2"
42
- DEFAULT_NEMOTRON_RERANKER_MODEL = "nvidia/llama-nemotron-rerank-1b-v2"
43
  OPENAI_REASONING_EFFORT = "medium"
44
  ANTHROPIC_THINKING_BUDGET = 2048
45
  DEFAULT_CHUNK_SIZE = 350
@@ -53,12 +50,11 @@ TOP_K_RETRIEVAL = 5
53
  MAX_UPLOAD_SIZE_MB = 25
54
  TIMEOUT_SECONDS = 30
55
 
56
- ProviderLiteral = Literal["nemotron", "qwen", "openai", "anthropic", "gemini", "cohere"]
57
 
58
  # Conservative full-document QA input budgets derived from provider/model
59
  # context-window docs, with headroom reserved for prompts and outputs.
60
  PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS: dict[ProviderLiteral, int] = {
61
- "nemotron": 64_000,
62
  "qwen": 24_000,
63
  "openai": 900_000,
64
  "anthropic": 900_000,
@@ -89,26 +85,9 @@ GEMINI_API_KEY: Optional[str] = _read_env_key("GEMINI_API_KEY")
89
  COHERE_API_KEY: Optional[str] = _read_env_key("COHERE_API_KEY")
90
  DEFAULT_COHERE_KEY: Optional[str] = _read_env_key("DEFAULT_COHERE_KEY")
91
  HF_TOKEN: Optional[str] = _read_env_key("HF_TOKEN")
92
- NEMOTRON_API_KEY: Optional[str] = _read_env_key("NEMOTRON_API_KEY") or HF_TOKEN
93
- NEMOTRON_BASE_URL: str = os.getenv("NEMOTRON_BASE_URL", "https://router.huggingface.co/v1").strip() or "https://router.huggingface.co/v1"
94
- NEMOTRON_RETRIEVER_API_KEY: Optional[str] = _read_env_key("NEMOTRON_RETRIEVER_API_KEY") or NEMOTRON_API_KEY
95
- NEMOTRON_RETRIEVER_BASE_URL: str = (
96
- os.getenv("NEMOTRON_RETRIEVER_BASE_URL", NEMOTRON_BASE_URL).strip() or NEMOTRON_BASE_URL
97
- )
98
- NEMOTRON_RERANKER_API_KEY: Optional[str] = _read_env_key("NEMOTRON_RERANKER_API_KEY") or NEMOTRON_API_KEY
99
- NEMOTRON_RERANKER_URL: Optional[str] = _read_env_key("NEMOTRON_RERANKER_URL")
100
 
101
 
102
  PROVIDER_METADATA: list[ProviderConfig] = [
103
- ProviderConfig(
104
- name="nemotron",
105
- key_prefix=None,
106
- display_name="Nemotron Nano 30B",
107
- instructions=(
108
- "Use NEMOTRON_API_KEY for a self-hosted or external OpenAI-compatible Nemotron endpoint. "
109
- "This is optional and no longer the default app path."
110
- ),
111
- ),
112
  ProviderConfig(
113
  name="qwen",
114
  key_prefix=None,
 
32
  category=FutureWarning,
33
  )
34
 
35
+ SUPPORTED_PROVIDERS = ["qwen", "openai", "anthropic", "gemini", "cohere"]
36
  DEFAULT_PROVIDER: str = "qwen"
 
37
  DEFAULT_QWEN_MODEL = "Qwen/Qwen3-14B:cheapest"
38
  DEFAULT_CHUNK_TOKENIZER_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
39
  DEFAULT_FALLBACK_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
 
 
40
  OPENAI_REASONING_EFFORT = "medium"
41
  ANTHROPIC_THINKING_BUDGET = 2048
42
  DEFAULT_CHUNK_SIZE = 350
 
50
  MAX_UPLOAD_SIZE_MB = 25
51
  TIMEOUT_SECONDS = 30
52
 
53
+ ProviderLiteral = Literal["qwen", "openai", "anthropic", "gemini", "cohere"]
54
 
55
  # Conservative full-document QA input budgets derived from provider/model
56
  # context-window docs, with headroom reserved for prompts and outputs.
57
  PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS: dict[ProviderLiteral, int] = {
 
58
  "qwen": 24_000,
59
  "openai": 900_000,
60
  "anthropic": 900_000,
 
85
  COHERE_API_KEY: Optional[str] = _read_env_key("COHERE_API_KEY")
86
  DEFAULT_COHERE_KEY: Optional[str] = _read_env_key("DEFAULT_COHERE_KEY")
87
  HF_TOKEN: Optional[str] = _read_env_key("HF_TOKEN")
 
 
 
 
 
 
 
 
88
 
89
 
90
  PROVIDER_METADATA: list[ProviderConfig] = [
 
 
 
 
 
 
 
 
 
91
  ProviderConfig(
92
  name="qwen",
93
  key_prefix=None,
modal_nemotron_service.py DELETED
@@ -1,165 +0,0 @@
1
- """Modal deployment for an OpenAI-compatible Nemotron generator endpoint.
2
-
3
- This service solves the current Hugging Face router gap for
4
- `nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16` by serving the model directly on
5
- Modal with SGLang. The app can then point `NEMOTRON_BASE_URL` at the deployed
6
- Modal URL and continue using the existing OpenAI client path.
7
- """
8
-
9
- from __future__ import annotations
10
-
11
- import os
12
- import subprocess
13
- import time
14
-
15
- import modal
16
- import modal.experimental
17
-
18
- APP_NAME = os.getenv("MODAL_APP_NAME", "legislation-explainer-nemotron")
19
- MODEL_NAME = os.getenv(
20
- "NEMOTRON_MODAL_MODEL",
21
- "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
22
- )
23
- PORT = int(os.getenv("NEMOTRON_MODAL_PORT", "8000"))
24
- GPU = os.getenv("NEMOTRON_MODAL_GPU", "H100")
25
- REGION = os.getenv("NEMOTRON_MODAL_REGION", "us-east")
26
- PROXY_REGION = os.getenv("NEMOTRON_MODAL_PROXY_REGION", "us-east")
27
- MIN_CONTAINERS = int(os.getenv("NEMOTRON_MODAL_MIN_CONTAINERS", "0"))
28
- SCALEDOWN_WINDOW = int(os.getenv("NEMOTRON_MODAL_SCALEDOWN_WINDOW", "300"))
29
- TIMEOUT_SECONDS = int(os.getenv("NEMOTRON_MODAL_TIMEOUT_SECONDS", "3600"))
30
- STARTUP_TIMEOUT_SECONDS = int(os.getenv("NEMOTRON_MODAL_STARTUP_TIMEOUT_SECONDS", "3600"))
31
- TARGET_INPUTS = int(os.getenv("NEMOTRON_MODAL_TARGET_INPUTS", "8"))
32
- HF_SECRET_NAME = os.getenv("NEMOTRON_MODAL_HF_SECRET_NAME", "huggingface-secret")
33
- VOLUME_NAME = os.getenv("NEMOTRON_MODAL_VOLUME_NAME", "legislation-explainer-nemotron-cache")
34
- HF_CACHE_PATH = "/root/.cache/huggingface"
35
-
36
- server_image = (
37
- modal.Image.from_registry("lmsysorg/sglang:v0.5.11")
38
- .entrypoint([])
39
- .run_commands("rm -rf /root/.cache/huggingface")
40
- .env(
41
- {
42
- "HF_HUB_CACHE": HF_CACHE_PATH,
43
- "HF_XET_HIGH_PERFORMANCE": "1",
44
- "SAFETENSORS_FAST_GPU": "1",
45
- "NVIDIA_TF32_OVERRIDE": "1",
46
- "SGLANG_ENABLE_SPEC_V2": "1",
47
- "SGLANG_ENABLE_JIT_DEEPGEMM": "0",
48
- }
49
- )
50
- )
51
-
52
- with server_image.imports():
53
- import requests
54
-
55
- app = modal.App(APP_NAME, image=server_image)
56
- hf_secret = modal.Secret.from_name(HF_SECRET_NAME)
57
- hf_cache_volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True)
58
-
59
-
60
- def _check_running(process: subprocess.Popen[str]) -> None:
61
- return_code = process.poll()
62
- if return_code is not None:
63
- raise subprocess.CalledProcessError(return_code, cmd=process.args)
64
-
65
-
66
- def _wait_ready(process: subprocess.Popen[str], timeout_seconds: int = 1800) -> None:
67
- deadline = time.time() + timeout_seconds
68
- while time.time() < deadline:
69
- try:
70
- _check_running(process)
71
- response = requests.get(f"http://127.0.0.1:{PORT}/health", timeout=5)
72
- response.raise_for_status()
73
- return
74
- except (
75
- requests.exceptions.ConnectionError,
76
- requests.exceptions.HTTPError,
77
- requests.exceptions.Timeout,
78
- subprocess.CalledProcessError,
79
- ):
80
- time.sleep(5)
81
- raise TimeoutError(f"SGLang server was not ready within {timeout_seconds} seconds.")
82
-
83
-
84
- def _warm_up_server() -> None:
85
- payload = {
86
- "model": MODEL_NAME,
87
- "messages": [{"role": "user", "content": "Reply with the single word READY."}],
88
- "max_tokens": 8,
89
- "temperature": 0,
90
- }
91
- for _ in range(2):
92
- response = requests.post(
93
- f"http://127.0.0.1:{PORT}/v1/chat/completions",
94
- json=payload,
95
- timeout=30,
96
- )
97
- response.raise_for_status()
98
-
99
-
100
- @app.cls(
101
- gpu=GPU,
102
- min_containers=MIN_CONTAINERS,
103
- scaledown_window=SCALEDOWN_WINDOW,
104
- timeout=TIMEOUT_SECONDS,
105
- startup_timeout=STARTUP_TIMEOUT_SECONDS,
106
- region=REGION,
107
- secrets=[hf_secret],
108
- volumes={HF_CACHE_PATH: hf_cache_volume},
109
- )
110
- @modal.experimental.http_server(
111
- port=PORT,
112
- proxy_regions=[PROXY_REGION],
113
- exit_grace_period=15,
114
- )
115
- @modal.concurrent(target_inputs=TARGET_INPUTS)
116
- class NemotronGenerator:
117
- """Serve Nemotron through SGLang's OpenAI-compatible HTTP API."""
118
-
119
- process: subprocess.Popen[str] | None = None
120
-
121
- @modal.enter()
122
- def start_server(self) -> None:
123
- command = [
124
- "sglang",
125
- "serve",
126
- "--model-path",
127
- MODEL_NAME,
128
- "--served-model-name",
129
- MODEL_NAME,
130
- "--host",
131
- "0.0.0.0",
132
- "--port",
133
- str(PORT),
134
- "--trust-remote-code",
135
- "--reasoning-parser",
136
- "nemotron_3",
137
- "--context-length",
138
- os.getenv("NEMOTRON_MODAL_CONTEXT_LENGTH", "32768"),
139
- "--chunked-prefill-size",
140
- os.getenv("NEMOTRON_MODAL_CHUNKED_PREFILL_SIZE", "8192"),
141
- "--mem-fraction-static",
142
- os.getenv("NEMOTRON_MODAL_MEM_FRACTION_STATIC", "0.82"),
143
- ]
144
- self.process = subprocess.Popen(command)
145
- _wait_ready(self.process)
146
- _warm_up_server()
147
-
148
- @modal.exit()
149
- def stop_server(self) -> None:
150
- if self.process is None:
151
- return
152
- if self.process.poll() is None:
153
- self.process.terminate()
154
- try:
155
- self.process.wait(timeout=20)
156
- except subprocess.TimeoutExpired:
157
- self.process.kill()
158
-
159
-
160
- @app.local_entrypoint()
161
- def main() -> None:
162
- print(
163
- "Deploy with `modal deploy modal_nemotron_service.py`. "
164
- "Then set NEMOTRON_BASE_URL to the deployed service URL plus `/v1`."
165
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
services/providers.py CHANGED
@@ -13,12 +13,9 @@ from config import (
13
  COHERE_API_KEY,
14
  DEFAULT_PROVIDER,
15
  DEFAULT_COHERE_KEY,
16
- DEFAULT_NEMOTRON_MODEL,
17
  GEMINI_API_KEY,
18
  DEFAULT_QWEN_MODEL,
19
  HF_TOKEN,
20
- NEMOTRON_API_KEY,
21
- NEMOTRON_BASE_URL,
22
  OPENAI_API_KEY,
23
  PROVIDER_METADATA,
24
  ProviderConfig,
@@ -37,7 +34,6 @@ class ProviderClient:
37
 
38
 
39
  _PROVIDER_MODEL_MAP: Dict[ProviderLiteral, str] = {
40
- "nemotron": DEFAULT_NEMOTRON_MODEL,
41
  "qwen": DEFAULT_QWEN_MODEL,
42
  "openai": "gpt-5.5",
43
  "anthropic": "claude-sonnet-4-6",
@@ -47,7 +43,6 @@ _PROVIDER_MODEL_MAP: Dict[ProviderLiteral, str] = {
47
 
48
 
49
  _DEFAULT_PROVIDER_KEYS: Dict[ProviderLiteral, Optional[str]] = {
50
- "nemotron": NEMOTRON_API_KEY or HF_TOKEN,
51
  "qwen": HF_TOKEN,
52
  "openai": OPENAI_API_KEY,
53
  "anthropic": ANTHROPIC_API_KEY,
@@ -84,9 +79,7 @@ def validate_api_key(provider: ProviderLiteral, api_key: str | None) -> tuple[bo
84
 
85
 
86
  def instantiate_client(provider: ProviderLiteral, api_key: str) -> ProviderClient:
87
- if provider == "nemotron":
88
- client = OpenAI(api_key=api_key, base_url=NEMOTRON_BASE_URL)
89
- elif provider == "qwen":
90
  client = OpenAI(api_key=api_key, base_url="https://router.huggingface.co/v1")
91
  elif provider == "openai":
92
  client = OpenAI(api_key=api_key)
 
13
  COHERE_API_KEY,
14
  DEFAULT_PROVIDER,
15
  DEFAULT_COHERE_KEY,
 
16
  GEMINI_API_KEY,
17
  DEFAULT_QWEN_MODEL,
18
  HF_TOKEN,
 
 
19
  OPENAI_API_KEY,
20
  PROVIDER_METADATA,
21
  ProviderConfig,
 
34
 
35
 
36
  _PROVIDER_MODEL_MAP: Dict[ProviderLiteral, str] = {
 
37
  "qwen": DEFAULT_QWEN_MODEL,
38
  "openai": "gpt-5.5",
39
  "anthropic": "claude-sonnet-4-6",
 
43
 
44
 
45
  _DEFAULT_PROVIDER_KEYS: Dict[ProviderLiteral, Optional[str]] = {
 
46
  "qwen": HF_TOKEN,
47
  "openai": OPENAI_API_KEY,
48
  "anthropic": ANTHROPIC_API_KEY,
 
79
 
80
 
81
  def instantiate_client(provider: ProviderLiteral, api_key: str) -> ProviderClient:
82
+ if provider == "qwen":
 
 
83
  client = OpenAI(api_key=api_key, base_url="https://router.huggingface.co/v1")
84
  elif provider == "openai":
85
  client = OpenAI(api_key=api_key)
services/rag_pipeline.py CHANGED
@@ -5,7 +5,6 @@ from __future__ import annotations
5
  import hashlib
6
  import inspect
7
  import json
8
- import math
9
  import re
10
  from concurrent.futures import ThreadPoolExecutor
11
  from dataclasses import dataclass, field
@@ -33,9 +32,7 @@ from langchain_community.embeddings import SentenceTransformerEmbeddings
33
  from langchain_community.vectorstores import FAISS
34
  from langchain_core.documents import Document
35
  from langchain.text_splitter import RecursiveCharacterTextSplitter
36
- from openai import BadRequestError, OpenAI
37
  from pydantic import BaseModel, Field
38
- import httpx
39
  from transformers import AutoTokenizer
40
 
41
  from config import (
@@ -44,13 +41,6 @@ from config import (
44
  DEFAULT_CHUNK_SIZE,
45
  DEFAULT_CHUNK_TOKENIZER_MODEL,
46
  DEFAULT_FALLBACK_EMBEDDING_MODEL,
47
- DEFAULT_QWEN_MODEL,
48
- DEFAULT_NEMOTRON_RETRIEVER_MODEL,
49
- DEFAULT_NEMOTRON_RERANKER_MODEL,
50
- NEMOTRON_RETRIEVER_API_KEY,
51
- NEMOTRON_RETRIEVER_BASE_URL,
52
- NEMOTRON_RERANKER_API_KEY,
53
- NEMOTRON_RERANKER_URL,
54
  OPENAI_REASONING_EFFORT,
55
  PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS,
56
  SCAN_BATCH_SIZE,
@@ -58,7 +48,6 @@ from config import (
58
  SCAN_CHUNK_SIZE,
59
  SCAN_MAX_WINDOWS,
60
  SCAN_TOP_K,
61
- TIMEOUT_SECONDS,
62
  TOP_K_RETRIEVAL,
63
  )
64
  from services.precomputed_assets import PrecomputedExampleAsset, load_precomputed_asset_for_url
@@ -139,14 +128,6 @@ class AnalysisSnippet:
139
  section: str
140
  text: str
141
 
142
-
143
- @dataclass
144
- class NemotronRetrievalIndex:
145
- chunks: list[str]
146
- embeddings: list[list[float]]
147
- model_name: str = DEFAULT_NEMOTRON_RETRIEVER_MODEL
148
-
149
-
150
  @dataclass
151
  class CachedDocumentArtifacts:
152
  document_hash: str
@@ -162,7 +143,7 @@ class CachedDocumentArtifacts:
162
 
163
 
164
  _DOCUMENT_CACHE: dict[str, CachedDocumentArtifacts] = {}
165
- RetrievalIndex = FAISS | NemotronRetrievalIndex
166
 
167
 
168
  @lru_cache(maxsize=1)
@@ -177,8 +158,7 @@ def _embedding_tokenizer(model_name: str = DEFAULT_CHUNK_TOKENIZER_MODEL) -> Tok
177
 
178
  def warm_embedding_stack(model_name: str = DEFAULT_CHUNK_TOKENIZER_MODEL) -> None:
179
  _embedding_tokenizer(model_name)
180
- if not _nemotron_retrieval_ready():
181
- _fallback_embedding_model()
182
 
183
 
184
  def _split_with_tokenizer(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
@@ -233,60 +213,6 @@ def _anthropic_thinking_kwargs(provider_client: ProviderClient) -> dict[str, Any
233
  }
234
 
235
 
236
- def _fallback_generation_model(provider_client: ProviderClient) -> str | None:
237
- if provider_client.name == "nemotron":
238
- return DEFAULT_QWEN_MODEL
239
- return None
240
-
241
-
242
- def _is_model_not_supported_error(exc: BadRequestError) -> bool:
243
- body = getattr(exc, "body", None)
244
- if isinstance(body, dict):
245
- error_block = body.get("error")
246
- if isinstance(error_block, dict):
247
- code = str(error_block.get("code", "")).strip().lower()
248
- message = str(error_block.get("message", "")).strip().lower()
249
- else:
250
- code = str(body.get("code", "")).strip().lower()
251
- message = str(body.get("message", "")).strip().lower()
252
- if code == "model_not_supported":
253
- return True
254
- if "not supported by any provider" in message:
255
- return True
256
- return False
257
-
258
-
259
- def _chat_completion_with_fallback(
260
- provider_client: ProviderClient,
261
- *,
262
- messages: list[dict[str, str]],
263
- temperature: float = 0,
264
- ) -> Any:
265
- candidate_models = [provider_client.default_model]
266
- fallback_model = _fallback_generation_model(provider_client)
267
- if fallback_model and fallback_model not in candidate_models:
268
- candidate_models.append(fallback_model)
269
-
270
- last_error: Exception | None = None
271
- for index, model_name in enumerate(candidate_models):
272
- try:
273
- return provider_client.client.chat.completions.create(
274
- model=model_name,
275
- messages=messages,
276
- temperature=temperature,
277
- )
278
- except BadRequestError as exc:
279
- last_error = exc
280
- is_last_model = index == len(candidate_models) - 1
281
- if not is_last_model and _is_model_not_supported_error(exc):
282
- continue
283
- raise
284
-
285
- if last_error is not None:
286
- raise last_error
287
- raise RuntimeError("No candidate generation models were available.")
288
-
289
-
290
  def split_into_chunks(text: str, chunk_size: int = DEFAULT_CHUNK_SIZE, chunk_overlap: int = DEFAULT_CHUNK_OVERLAP) -> List[str]:
291
  return _split_with_tokenizer(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
292
 
@@ -306,39 +232,8 @@ def build_vector_store(chunks: Iterable[str]) -> FAISS:
306
  return FAISS.from_documents(docs, embedding=embeddings)
307
 
308
 
309
- def _nemotron_retrieval_ready() -> bool:
310
- return bool(NEMOTRON_RETRIEVER_BASE_URL and NEMOTRON_RETRIEVER_API_KEY and DEFAULT_NEMOTRON_RETRIEVER_MODEL)
311
-
312
-
313
- def _nemotron_reranker_ready() -> bool:
314
- return bool(NEMOTRON_RERANKER_URL and NEMOTRON_RERANKER_API_KEY and DEFAULT_NEMOTRON_RERANKER_MODEL)
315
-
316
-
317
- def _embed_texts_with_nemotron(texts: list[str]) -> list[list[float]]:
318
- if not texts:
319
- return []
320
- if not _nemotron_retrieval_ready():
321
- raise RuntimeError("Nemotron retriever endpoint is not configured.")
322
-
323
- client = OpenAI(api_key=NEMOTRON_RETRIEVER_API_KEY, base_url=NEMOTRON_RETRIEVER_BASE_URL)
324
- response = client.embeddings.create(model=DEFAULT_NEMOTRON_RETRIEVER_MODEL, input=texts)
325
- ordered = sorted(response.data, key=lambda item: item.index)
326
- return [list(item.embedding) for item in ordered]
327
-
328
-
329
- def build_nemotron_retrieval_index(chunks: Iterable[str]) -> NemotronRetrievalIndex:
330
- chunk_list = list(chunks)
331
- return NemotronRetrievalIndex(chunks=chunk_list, embeddings=_embed_texts_with_nemotron(chunk_list))
332
-
333
-
334
  def build_retrieval_index(chunks: Iterable[str]) -> RetrievalIndex:
335
- chunk_list = list(chunks)
336
- if _nemotron_retrieval_ready():
337
- try:
338
- return build_nemotron_retrieval_index(chunk_list)
339
- except Exception:
340
- pass
341
- return build_vector_store(chunk_list)
342
 
343
 
344
  def document_hash_for_text(text: str) -> str:
@@ -346,15 +241,7 @@ def document_hash_for_text(text: str) -> str:
346
 
347
 
348
  def _retrieval_signature() -> str:
349
- return "|".join(
350
- [
351
- DEFAULT_CHUNK_TOKENIZER_MODEL,
352
- DEFAULT_FALLBACK_EMBEDDING_MODEL,
353
- DEFAULT_NEMOTRON_RETRIEVER_MODEL,
354
- DEFAULT_NEMOTRON_RERANKER_MODEL if _nemotron_reranker_ready() else "no-rerank",
355
- "nemotron" if _nemotron_retrieval_ready() else "fallback",
356
- ]
357
- )
358
 
359
 
360
  def document_cache_key(document_hash: str) -> str:
@@ -544,9 +431,9 @@ def _generate_json_payload(
544
  **_openai_reasoning_kwargs(provider_client),
545
  )
546
  return _extract_openai_json(response)
547
- if provider_client.name in {"nemotron", "qwen"}:
548
- response = _chat_completion_with_fallback(
549
- provider_client,
550
  messages=[
551
  {"role": "system", "content": prompt},
552
  {
@@ -966,9 +853,9 @@ def _answer_question_from_full_document(
966
  **_openai_reasoning_kwargs(provider_client),
967
  )
968
  return _extract_openai_text(response)
969
- if provider_client.name in {"nemotron", "qwen"}:
970
- response = _chat_completion_with_fallback(
971
- provider_client,
972
  messages=[
973
  {"role": "system", "content": prompt},
974
  {"role": "user", "content": user_text},
@@ -1008,75 +895,14 @@ def _answer_question_from_full_document(
1008
  return _extract_gemini_text(response)
1009
 
1010
 
1011
- def _cosine_similarity(left: list[float], right: list[float]) -> float:
1012
- dot_product = sum(a * b for a, b in zip(left, right))
1013
- left_norm = math.sqrt(sum(value * value for value in left))
1014
- right_norm = math.sqrt(sum(value * value for value in right))
1015
- if left_norm == 0 or right_norm == 0:
1016
- return 0.0
1017
- return dot_product / (left_norm * right_norm)
1018
-
1019
-
1020
- def _rerank_candidates(question: str, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
1021
- if not candidates or not _nemotron_reranker_ready():
1022
- return candidates
1023
-
1024
- headers = {"Content-Type": "application/json"}
1025
- if NEMOTRON_RERANKER_API_KEY:
1026
- headers["Authorization"] = f"Bearer {NEMOTRON_RERANKER_API_KEY}"
1027
-
1028
- payload = {
1029
- "model": DEFAULT_NEMOTRON_RERANKER_MODEL,
1030
- "query": question,
1031
- "documents": [candidate["text"] for candidate in candidates],
1032
- "top_n": len(candidates),
1033
- }
1034
- try:
1035
- response = httpx.post(NEMOTRON_RERANKER_URL, json=payload, headers=headers, timeout=TIMEOUT_SECONDS)
1036
- response.raise_for_status()
1037
- body = response.json()
1038
- except Exception:
1039
- return candidates
1040
-
1041
- results = body.get("results") or body.get("data") or []
1042
- if not isinstance(results, list):
1043
- return candidates
1044
-
1045
- ranked_candidates: list[tuple[float, int, dict[str, Any]]] = []
1046
- for item in results:
1047
- if not isinstance(item, dict):
1048
- continue
1049
- index = item.get("index")
1050
- if not isinstance(index, int) or index < 0 or index >= len(candidates):
1051
- continue
1052
- score = item.get("relevance_score", item.get("score", 0.0))
1053
- try:
1054
- numeric_score = float(score)
1055
- except (TypeError, ValueError):
1056
- numeric_score = 0.0
1057
- ranked_candidates.append((numeric_score, index, candidates[index]))
1058
-
1059
- if not ranked_candidates:
1060
- return candidates
1061
- ranked_candidates.sort(key=lambda item: (-item[0], item[1]))
1062
- return [candidate for _, _, candidate in ranked_candidates]
1063
-
1064
-
1065
  def _retrieve_context_from_faiss(vector_store: FAISS, question: str) -> tuple[str, list[Citation]]:
1066
- retrieved_docs = vector_store.similarity_search(question, k=max(TOP_K_RETRIEVAL * 2, TOP_K_RETRIEVAL))
1067
- candidates = [
1068
- {
1069
- "chunk_id": idx,
1070
- "text": doc.page_content.strip(),
1071
- }
1072
- for idx, doc in enumerate(retrieved_docs, start=1)
1073
- if doc.page_content.strip()
1074
- ]
1075
- ranked_candidates = _rerank_candidates(question, candidates)[:TOP_K_RETRIEVAL]
1076
  context_blocks: list[str] = []
1077
  citations: list[Citation] = []
1078
- for idx, candidate in enumerate(ranked_candidates, start=1):
1079
- snippet = candidate["text"]
 
 
 
1080
  context_blocks.append(f"[{idx}] {snippet}")
1081
  citations.append(Citation(ref_id=idx, snippet=snippet))
1082
 
@@ -1084,33 +910,7 @@ def _retrieve_context_from_faiss(vector_store: FAISS, question: str) -> tuple[st
1084
  return context_text, citations
1085
 
1086
 
1087
- def _retrieve_context_from_nemotron_index(vector_store: NemotronRetrievalIndex, question: str) -> tuple[str, list[Citation]]:
1088
- query_embedding = _embed_texts_with_nemotron([question])[0]
1089
- scored_candidates = [
1090
- {
1091
- "chunk_id": idx,
1092
- "text": chunk.strip(),
1093
- "score": _cosine_similarity(query_embedding, embedding),
1094
- }
1095
- for idx, (chunk, embedding) in enumerate(zip(vector_store.chunks, vector_store.embeddings), start=1)
1096
- if chunk.strip()
1097
- ]
1098
- scored_candidates.sort(key=lambda item: (-item["score"], item["chunk_id"]))
1099
- ranked_candidates = _rerank_candidates(question, scored_candidates[: max(TOP_K_RETRIEVAL * 3, TOP_K_RETRIEVAL)])[:TOP_K_RETRIEVAL]
1100
-
1101
- context_blocks: list[str] = []
1102
- citations: list[Citation] = []
1103
- for ref_id, candidate in enumerate(ranked_candidates, start=1):
1104
- snippet = candidate["text"]
1105
- citations.append(Citation(ref_id=ref_id, snippet=snippet))
1106
- context_blocks.append(f"[{ref_id}] {snippet}")
1107
- context_text = "\n\n".join(context_blocks) if context_blocks else "No context available."
1108
- return context_text, citations
1109
-
1110
-
1111
  def _retrieve_context_from_index(vector_store: RetrievalIndex, question: str) -> tuple[str, list[Citation]]:
1112
- if isinstance(vector_store, NemotronRetrievalIndex):
1113
- return _retrieve_context_from_nemotron_index(vector_store, question)
1114
  return _retrieve_context_from_faiss(vector_store, question)
1115
 
1116
 
@@ -1137,9 +937,9 @@ def _generate_answer_from_context(provider_client: ProviderClient, context_text:
1137
  **_openai_reasoning_kwargs(provider_client),
1138
  )
1139
  answer = _extract_openai_text(response)
1140
- elif provider_client.name in {"nemotron", "qwen"}:
1141
- response = _chat_completion_with_fallback(
1142
- provider_client,
1143
  messages=[
1144
  {"role": "system", "content": prompt},
1145
  {
@@ -1310,9 +1110,9 @@ def _scan_batch(provider_client: ProviderClient, batch: list[ScanChunk], questio
1310
  **_openai_reasoning_kwargs(provider_client),
1311
  )
1312
  return _extract_openai_json(response)
1313
- if provider_client.name in {"nemotron", "qwen"}:
1314
- response = _chat_completion_with_fallback(
1315
- provider_client,
1316
  messages=[
1317
  {"role": "system", "content": prompt},
1318
  {"role": "user", "content": user_text},
 
5
  import hashlib
6
  import inspect
7
  import json
 
8
  import re
9
  from concurrent.futures import ThreadPoolExecutor
10
  from dataclasses import dataclass, field
 
32
  from langchain_community.vectorstores import FAISS
33
  from langchain_core.documents import Document
34
  from langchain.text_splitter import RecursiveCharacterTextSplitter
 
35
  from pydantic import BaseModel, Field
 
36
  from transformers import AutoTokenizer
37
 
38
  from config import (
 
41
  DEFAULT_CHUNK_SIZE,
42
  DEFAULT_CHUNK_TOKENIZER_MODEL,
43
  DEFAULT_FALLBACK_EMBEDDING_MODEL,
 
 
 
 
 
 
 
44
  OPENAI_REASONING_EFFORT,
45
  PROVIDER_FULL_DOCUMENT_QA_TOKEN_BUDGETS,
46
  SCAN_BATCH_SIZE,
 
48
  SCAN_CHUNK_SIZE,
49
  SCAN_MAX_WINDOWS,
50
  SCAN_TOP_K,
 
51
  TOP_K_RETRIEVAL,
52
  )
53
  from services.precomputed_assets import PrecomputedExampleAsset, load_precomputed_asset_for_url
 
128
  section: str
129
  text: str
130
 
 
 
 
 
 
 
 
 
131
  @dataclass
132
  class CachedDocumentArtifacts:
133
  document_hash: str
 
143
 
144
 
145
  _DOCUMENT_CACHE: dict[str, CachedDocumentArtifacts] = {}
146
+ RetrievalIndex = FAISS
147
 
148
 
149
  @lru_cache(maxsize=1)
 
158
 
159
  def warm_embedding_stack(model_name: str = DEFAULT_CHUNK_TOKENIZER_MODEL) -> None:
160
  _embedding_tokenizer(model_name)
161
+ _fallback_embedding_model()
 
162
 
163
 
164
  def _split_with_tokenizer(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
 
213
  }
214
 
215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
216
  def split_into_chunks(text: str, chunk_size: int = DEFAULT_CHUNK_SIZE, chunk_overlap: int = DEFAULT_CHUNK_OVERLAP) -> List[str]:
217
  return _split_with_tokenizer(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
218
 
 
232
  return FAISS.from_documents(docs, embedding=embeddings)
233
 
234
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  def build_retrieval_index(chunks: Iterable[str]) -> RetrievalIndex:
236
+ return build_vector_store(list(chunks))
 
 
 
 
 
 
237
 
238
 
239
  def document_hash_for_text(text: str) -> str:
 
241
 
242
 
243
  def _retrieval_signature() -> str:
244
+ return "|".join([DEFAULT_CHUNK_TOKENIZER_MODEL, DEFAULT_FALLBACK_EMBEDDING_MODEL, "faiss"])
 
 
 
 
 
 
 
 
245
 
246
 
247
  def document_cache_key(document_hash: str) -> str:
 
431
  **_openai_reasoning_kwargs(provider_client),
432
  )
433
  return _extract_openai_json(response)
434
+ if provider_client.name == "qwen":
435
+ response = provider_client.client.chat.completions.create(
436
+ model=provider_client.default_model,
437
  messages=[
438
  {"role": "system", "content": prompt},
439
  {
 
853
  **_openai_reasoning_kwargs(provider_client),
854
  )
855
  return _extract_openai_text(response)
856
+ if provider_client.name == "qwen":
857
+ response = provider_client.client.chat.completions.create(
858
+ model=provider_client.default_model,
859
  messages=[
860
  {"role": "system", "content": prompt},
861
  {"role": "user", "content": user_text},
 
895
  return _extract_gemini_text(response)
896
 
897
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
898
  def _retrieve_context_from_faiss(vector_store: FAISS, question: str) -> tuple[str, list[Citation]]:
 
 
 
 
 
 
 
 
 
 
899
  context_blocks: list[str] = []
900
  citations: list[Citation] = []
901
+ retrieved_docs = vector_store.similarity_search(question, k=TOP_K_RETRIEVAL)
902
+ for idx, doc in enumerate(retrieved_docs, start=1):
903
+ snippet = doc.page_content.strip()
904
+ if not snippet:
905
+ continue
906
  context_blocks.append(f"[{idx}] {snippet}")
907
  citations.append(Citation(ref_id=idx, snippet=snippet))
908
 
 
910
  return context_text, citations
911
 
912
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
913
  def _retrieve_context_from_index(vector_store: RetrievalIndex, question: str) -> tuple[str, list[Citation]]:
 
 
914
  return _retrieve_context_from_faiss(vector_store, question)
915
 
916
 
 
937
  **_openai_reasoning_kwargs(provider_client),
938
  )
939
  answer = _extract_openai_text(response)
940
+ elif provider_client.name == "qwen":
941
+ response = provider_client.client.chat.completions.create(
942
+ model=provider_client.default_model,
943
  messages=[
944
  {"role": "system", "content": prompt},
945
  {
 
1110
  **_openai_reasoning_kwargs(provider_client),
1111
  )
1112
  return _extract_openai_json(response)
1113
+ if provider_client.name == "qwen":
1114
+ response = provider_client.client.chat.completions.create(
1115
+ model=provider_client.default_model,
1116
  messages=[
1117
  {"role": "system", "content": prompt},
1118
  {"role": "user", "content": user_text},
tests/test_rag_pipeline.py CHANGED
@@ -1,21 +1,16 @@
1
  from __future__ import annotations
2
 
3
  import json
4
-
5
- from openai import BadRequestError
6
-
7
- from config import DEFAULT_NEMOTRON_MODEL, DEFAULT_QWEN_MODEL, OPENAI_REASONING_EFFORT
8
  from services.providers import ProviderClient
9
  from services.providers import _PROVIDER_MODEL_MAP
10
  from services.rag_pipeline import (
11
  AnswerResult,
12
  AnalysisResult,
13
  Citation,
14
- NemotronRetrievalIndex,
15
  ScanMatch,
16
  _anthropic_thinking_kwargs,
17
  _analysis_answer_schema,
18
- _chat_completion_with_fallback,
19
  _full_document_budget,
20
  _analysis_schema,
21
  _extract_chat_completion_text,
@@ -224,7 +219,6 @@ def test_gemini_default_model_is_thinking_capable():
224
 
225
 
226
  def test_provider_defaults_use_thinking_capable_models():
227
- assert _PROVIDER_MODEL_MAP["nemotron"] == DEFAULT_NEMOTRON_MODEL
228
  assert _PROVIDER_MODEL_MAP["openai"] == "gpt-5.5"
229
  assert _PROVIDER_MODEL_MAP["anthropic"] == "claude-sonnet-4-6"
230
  assert _PROVIDER_MODEL_MAP["cohere"] == "command-a-reasoning-08-2025"
@@ -238,53 +232,6 @@ def test_extract_chat_completion_text_strips_qwen_thinking_block():
238
  assert _extract_chat_completion_text(response) == "Final answer"
239
 
240
 
241
- def test_chat_completion_with_fallback_uses_qwen_when_nemotron_is_unsupported():
242
- unsupported = BadRequestError(
243
- message="unsupported",
244
- response=type(
245
- "Response",
246
- (),
247
- {
248
- "request": None,
249
- "status_code": 400,
250
- "headers": {},
251
- "json": lambda self: {
252
- "error": {
253
- "message": "The requested model is not supported by any provider you have enabled.",
254
- "code": "model_not_supported",
255
- }
256
- },
257
- "text": "",
258
- },
259
- )(),
260
- body={
261
- "error": {
262
- "message": "The requested model is not supported by any provider you have enabled.",
263
- "code": "model_not_supported",
264
- }
265
- },
266
- )
267
- message = type("Message", (), {"content": "Fallback answer"})()
268
- choice = type("Choice", (), {"message": message})()
269
- success = type("Response", (), {"choices": [choice]})()
270
- client = DummyOpenAIChatClient([unsupported, success])
271
- provider_client = ProviderClient(
272
- name="nemotron",
273
- client=client,
274
- default_model=DEFAULT_NEMOTRON_MODEL,
275
- api_key="dummy-api-key",
276
- )
277
-
278
- response = _chat_completion_with_fallback(
279
- provider_client,
280
- messages=[{"role": "user", "content": "Question"}],
281
- temperature=0,
282
- )
283
-
284
- assert client.chat.completions.calls == [DEFAULT_NEMOTRON_MODEL, DEFAULT_QWEN_MODEL]
285
- assert response.choices[0].message.content == "Fallback answer"
286
-
287
-
288
  def test_answer_query_uses_scan_context(monkeypatch):
289
  provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
290
  analysis = AnalysisResult(executive_summary="Creates a new authority.")
@@ -402,34 +349,6 @@ def test_answer_query_from_full_document_uses_scan_when_full_document_too_large(
402
  assert result.citations[0].snippet == "licensing rule"
403
 
404
 
405
- def test_retrieve_context_from_nemotron_index_uses_reranked_candidates(monkeypatch):
406
- index = NemotronRetrievalIndex(
407
- chunks=[
408
- "Clause 1 sets startup licensing fees at a fixed annual amount.",
409
- "Clause 7 defines the minister's emergency powers.",
410
- "Clause 11 outlines reporting deadlines for data fiduciaries.",
411
- ],
412
- embeddings=[
413
- [0.2, 0.1, 0.9],
414
- [0.8, 0.1, 0.1],
415
- [0.1, 0.9, 0.2],
416
- ],
417
- )
418
-
419
- monkeypatch.setattr(
420
- "services.rag_pipeline._embed_texts_with_nemotron",
421
- lambda texts: [[0.0, 0.0, 1.0]],
422
- )
423
- monkeypatch.setattr(
424
- "services.rag_pipeline._rerank_candidates",
425
- lambda question, candidates: [candidates[2], candidates[0], candidates[1]],
426
- )
427
-
428
- context_text, citations = _retrieve_context_from_index(index, "What emergency powers does the minister have?")
429
- assert citations[0].snippet == "Clause 7 defines the minister's emergency powers."
430
- assert "[1] Clause 7 defines the minister's emergency powers." in context_text
431
-
432
-
433
  def test_answer_query_requests_deeper_answer_when_analysis_is_insufficient(monkeypatch):
434
  provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
435
  analysis = AnalysisResult(executive_summary="A short summary.")
 
1
  from __future__ import annotations
2
 
3
  import json
4
+ from config import DEFAULT_QWEN_MODEL, OPENAI_REASONING_EFFORT
 
 
 
5
  from services.providers import ProviderClient
6
  from services.providers import _PROVIDER_MODEL_MAP
7
  from services.rag_pipeline import (
8
  AnswerResult,
9
  AnalysisResult,
10
  Citation,
 
11
  ScanMatch,
12
  _anthropic_thinking_kwargs,
13
  _analysis_answer_schema,
 
14
  _full_document_budget,
15
  _analysis_schema,
16
  _extract_chat_completion_text,
 
219
 
220
 
221
  def test_provider_defaults_use_thinking_capable_models():
 
222
  assert _PROVIDER_MODEL_MAP["openai"] == "gpt-5.5"
223
  assert _PROVIDER_MODEL_MAP["anthropic"] == "claude-sonnet-4-6"
224
  assert _PROVIDER_MODEL_MAP["cohere"] == "command-a-reasoning-08-2025"
 
232
  assert _extract_chat_completion_text(response) == "Final answer"
233
 
234
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  def test_answer_query_uses_scan_context(monkeypatch):
236
  provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
237
  analysis = AnalysisResult(executive_summary="Creates a new authority.")
 
349
  assert result.citations[0].snippet == "licensing rule"
350
 
351
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
352
  def test_answer_query_requests_deeper_answer_when_analysis_is_insufficient(monkeypatch):
353
  provider_client = ProviderClient(name="openai", client=object(), default_model="dummy", api_key="dummy-api-key")
354
  analysis = AnalysisResult(executive_summary="A short summary.")