Spaces:
Sleeping
Sleeping
Commit ·
01b9f34
1
Parent(s): 385a823
fix: harden source display, subdirectory paths, and chunk ID uniqueness
Browse files- app_cli: _format_sources now includes SQL results in /sources output
- ingest: source_name uses full relative path (preserves subdirectories)
- ingest: chunk IDs use full path prefix to prevent collisions
- kb_meta: import _escape_braces from prompts instead of duplicating
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- app_cli.py +8 -2
- docs/plans/2026-03-05-sql-layer-design.md +0 -274
- docs/plans/2026-03-05-sql-layer-plan.md +0 -1719
- src/ingest.py +3 -2
- src/kb_meta.py +2 -5
- tests/__init__.py +0 -0
- tests/test_app_cli.py +0 -16
- tests/test_config_loader.py +0 -135
- tests/test_ingest.py +0 -53
- tests/test_kb_meta.py +0 -255
- tests/test_llm.py +0 -25
- tests/test_prompts.py +0 -111
- tests/test_query_engine.py +0 -172
- tests/test_readers.py +0 -47
- tests/test_readers_docx_excel_csv.py +0 -47
- tests/test_readers_stat.py +0 -38
- tests/test_retriever.py +0 -59
- tests/test_search.py +0 -28
- tests/test_setup.py +0 -39
- tests/test_sql_ingest.py +0 -204
- tests/test_sql_integration.py +0 -79
- tests/test_sql_retriever.py +0 -159
- tests/test_verifier.py +0 -54
app_cli.py
CHANGED
|
@@ -92,8 +92,9 @@ def _format_sources(retrieval_result: dict) -> str:
|
|
| 92 |
|
| 93 |
db_results = retrieval_result.get("db_results", [])
|
| 94 |
web_results = retrieval_result.get("web_results", [])
|
|
|
|
| 95 |
|
| 96 |
-
if not db_results and not web_results:
|
| 97 |
return "No sources were used for the last response."
|
| 98 |
|
| 99 |
if db_results:
|
|
@@ -105,9 +106,14 @@ def _format_sources(retrieval_result: dict) -> str:
|
|
| 105 |
dataset = meta.get("dataset", "")
|
| 106 |
lines.append(f" [{i}] {source}, page {page} [{dataset}]")
|
| 107 |
|
| 108 |
-
if
|
| 109 |
if db_results:
|
| 110 |
lines.append("")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
lines.append("Web Sources:")
|
| 112 |
for i, r in enumerate(web_results, 1):
|
| 113 |
year_str = f" ({r.get('year', '')})" if r.get("year") else ""
|
|
|
|
| 92 |
|
| 93 |
db_results = retrieval_result.get("db_results", [])
|
| 94 |
web_results = retrieval_result.get("web_results", [])
|
| 95 |
+
sql_results = retrieval_result.get("sql_results", [])
|
| 96 |
|
| 97 |
+
if not db_results and not web_results and not sql_results:
|
| 98 |
return "No sources were used for the last response."
|
| 99 |
|
| 100 |
if db_results:
|
|
|
|
| 106 |
dataset = meta.get("dataset", "")
|
| 107 |
lines.append(f" [{i}] {source}, page {page} [{dataset}]")
|
| 108 |
|
| 109 |
+
if sql_results:
|
| 110 |
if db_results:
|
| 111 |
lines.append("")
|
| 112 |
+
lines.append(f"SQL Results: {len(sql_results)} row(s) returned")
|
| 113 |
+
|
| 114 |
+
if web_results:
|
| 115 |
+
if db_results or sql_results:
|
| 116 |
+
lines.append("")
|
| 117 |
lines.append("Web Sources:")
|
| 118 |
for i, r in enumerate(web_results, 1):
|
| 119 |
year_str = f" ({r.get('year', '')})" if r.get("year") else ""
|
docs/plans/2026-03-05-sql-layer-design.md
DELETED
|
@@ -1,274 +0,0 @@
|
|
| 1 |
-
# SQL Layer for Tabular Datasets — Design Document
|
| 2 |
-
|
| 3 |
-
**Date:** 2026-03-05
|
| 4 |
-
**Status:** Approved
|
| 5 |
-
|
| 6 |
-
## Problem
|
| 7 |
-
|
| 8 |
-
The RAG chatbot uses ChromaDB (embedding similarity) + keyword search for retrieval. Tabular data (CSV, Excel, Stata, SPSS, R data) is chunked into text rows and embedded as vectors. This fails for structured queries like "PTS scores for China along the years" because embedding similarity cannot do filtering, aggregation, or precise lookups on structured data. Summarization queries ("how many countries are in the dataset?") are similarly impossible via vector search.
|
| 9 |
-
|
| 10 |
-
## Solution
|
| 11 |
-
|
| 12 |
-
Add a SQL layer alongside the existing vector retrieval. Tabular files ingested into `knowledge_base/` are loaded into both ChromaDB (for natural-language questions) and a SQLite database (for structured queries). The query understanding layer routes queries to the appropriate retrieval path.
|
| 13 |
-
|
| 14 |
-
## Architecture
|
| 15 |
-
|
| 16 |
-
```
|
| 17 |
-
User query
|
| 18 |
-
|
|
| 19 |
-
v
|
| 20 |
-
Query Understanding (extended)
|
| 21 |
-
| Outputs: action, route, search_query, display_query, sql_query
|
| 22 |
-
|
|
| 23 |
-
+-- route="sql" --------> retrieve_from_sql()
|
| 24 |
-
| |
|
| 25 |
-
| +-- results? --> format as context
|
| 26 |
-
| +-- empty? ----> fallback to retrieve_from_vectordb()
|
| 27 |
-
|
|
| 28 |
-
+-- route="vector" -----> retrieve_from_vectordb() (existing)
|
| 29 |
-
| |
|
| 30 |
-
| +-- results? --> format as context
|
| 31 |
-
| +-- empty? ----> fallback to retrieve_from_sql()
|
| 32 |
-
|
|
| 33 |
-
+-- route="both" -------> run both, merge results
|
| 34 |
-
|
|
| 35 |
-
v
|
| 36 |
-
Combined context (SQL text + vector chunks)
|
| 37 |
-
|
|
| 38 |
-
v
|
| 39 |
-
verify_and_respond() (unchanged 6-layer stack)
|
| 40 |
-
|
|
| 41 |
-
v
|
| 42 |
-
Cited response
|
| 43 |
-
```
|
| 44 |
-
|
| 45 |
-
**Key principle:** The SQL layer is a new retrieval path, not a new pipeline. Everything downstream (verification, citation, response formatting) stays the same.
|
| 46 |
-
|
| 47 |
-
## Design Decisions
|
| 48 |
-
|
| 49 |
-
| Decision | Choice | Rationale |
|
| 50 |
-
|---|---|---|
|
| 51 |
-
| Dual vs. exclusive ingestion | **Dual** — tabular files go to both ChromaDB and SQLite | Conceptual questions ("what does PTS measure?") still need vector search; metadata chunks stay in ChromaDB |
|
| 52 |
-
| Query routing | **LLM-based routing with fallback** | QU layer already makes an LLM call; extending it to output `route` is free. Fallback catches misroutes. |
|
| 53 |
-
| SQL engine | **SQLite** | Stdlib, zero-config, file-based. Matches the template's "download and run" philosophy. |
|
| 54 |
-
| Schema delivery to LLM | **Schema-in-prompt** | Researchers have 1–10 tables. Compact schema (~50 tokens/table) fits easily in QU prompt. Avoids extra LLM call. |
|
| 55 |
-
| SQL result handling | **Format as context, feed into verification pipeline** | Preserves anti-hallucination guarantees. SQL results are just another context source. |
|
| 56 |
-
| Summarization queries | **Always route to SQL** | Vector search cannot aggregate. QU prompt explicitly instructs: data summarization/counting/averaging/filtering → `sql` or `both`. |
|
| 57 |
-
|
| 58 |
-
## Ingestion Changes
|
| 59 |
-
|
| 60 |
-
During `python ingest.py`, tabular files get dual ingestion:
|
| 61 |
-
|
| 62 |
-
**Existing path (unchanged):** Read -> chunk -> embed -> ChromaDB
|
| 63 |
-
|
| 64 |
-
**New path (added):** Read -> load into SQLite as a table
|
| 65 |
-
|
| 66 |
-
### Table naming
|
| 67 |
-
|
| 68 |
-
`{dataset}_{filename}_{ext}` sanitized to valid SQL identifiers. Multi-sheet Excel files append sheet name.
|
| 69 |
-
|
| 70 |
-
```
|
| 71 |
-
knowledge_base/
|
| 72 |
-
PTS_dataset/
|
| 73 |
-
pts_data.csv -> SQLite table "pts_dataset__pts_data_csv"
|
| 74 |
-
codebook.pdf -> ChromaDB only (not tabular)
|
| 75 |
-
economic_data/
|
| 76 |
-
gdp.xlsx -> SQLite table "economic_data__gdp_xlsx_sheet1"
|
| 77 |
-
```
|
| 78 |
-
|
| 79 |
-
### Schema registry
|
| 80 |
-
|
| 81 |
-
After loading all tables, ingestion writes `sql_db/sql_schemas.json`:
|
| 82 |
-
|
| 83 |
-
```json
|
| 84 |
-
{
|
| 85 |
-
"pts_dataset__pts_data_csv": {
|
| 86 |
-
"source_file": "PTS_dataset/pts_data.csv",
|
| 87 |
-
"columns": [
|
| 88 |
-
{"name": "Country", "type": "TEXT", "sample": ["China", "India", "Brazil"]},
|
| 89 |
-
{"name": "Year", "type": "INTEGER", "sample": [2000, 2005, 2010]},
|
| 90 |
-
{"name": "PTS_A", "type": "REAL", "sample": [4.0, 3.0, 2.5]}
|
| 91 |
-
],
|
| 92 |
-
"row_count": 15234
|
| 93 |
-
}
|
| 94 |
-
}
|
| 95 |
-
```
|
| 96 |
-
|
| 97 |
-
### Type inference
|
| 98 |
-
|
| 99 |
-
During SQL ingestion, column types are inferred per-column:
|
| 100 |
-
- Try `int()` on all non-empty values -> INTEGER
|
| 101 |
-
- Try `float()` on all non-empty values -> REAL
|
| 102 |
-
- Otherwise -> TEXT
|
| 103 |
-
- Empty/NaN values -> NULL
|
| 104 |
-
|
| 105 |
-
### File locations
|
| 106 |
-
|
| 107 |
-
- SQLite DB: `sql_db/knowledge_base.db` (gitignored)
|
| 108 |
-
- Schema registry: `sql_db/sql_schemas.json` (gitignored)
|
| 109 |
-
- Tabular extensions triggering dual ingestion: `.csv`, `.tab`, `.tsv`, `.xlsx`, `.xls`, `.dta`, `.sav`, `.rds`, `.rda`
|
| 110 |
-
|
| 111 |
-
### Clearing
|
| 112 |
-
|
| 113 |
-
`ingest.py` already clears ChromaDB on re-run. It also drops all SQLite tables and regenerates `sql_schemas.json`.
|
| 114 |
-
|
| 115 |
-
## Query Understanding Changes
|
| 116 |
-
|
| 117 |
-
### Schema injection
|
| 118 |
-
|
| 119 |
-
When `sql_db/sql_schemas.json` exists, a compact summary is appended to the QU prompt:
|
| 120 |
-
|
| 121 |
-
```
|
| 122 |
-
Available SQL tables:
|
| 123 |
-
- pts_dataset__pts_data_csv (15234 rows): Country (TEXT), Year (INTEGER), PTS_A (REAL)
|
| 124 |
-
- economic_data__gdp_xlsx (5000 rows): Country (TEXT), Year (INTEGER), GDP (REAL)
|
| 125 |
-
```
|
| 126 |
-
|
| 127 |
-
~50 tokens per table. 10 tables = ~500 tokens. Schema text goes through `_escape_braces()` before prompt injection.
|
| 128 |
-
|
| 129 |
-
**Graceful degradation:** If `sql_schemas.json` does not exist (no tabular files ingested), schema injection is skipped, `route` always defaults to `"vector"`, and the pipeline behaves identically to the current system.
|
| 130 |
-
|
| 131 |
-
### Extended output format
|
| 132 |
-
|
| 133 |
-
The QU prompt instructs the LLM to return two new fields:
|
| 134 |
-
|
| 135 |
-
```json
|
| 136 |
-
{
|
| 137 |
-
"action": "search",
|
| 138 |
-
"route": "sql",
|
| 139 |
-
"search_query": "PTS scores China",
|
| 140 |
-
"display_query": "What are the PTS scores for China along the years?",
|
| 141 |
-
"sql_query": "SELECT Country, Year, PTS_A, PTS_S FROM pts_dataset__pts_data_csv WHERE Country = 'China' ORDER BY Year"
|
| 142 |
-
}
|
| 143 |
-
```
|
| 144 |
-
|
| 145 |
-
- `route`: `"sql"` | `"vector"` | `"both"` (default: `"vector"` if omitted — backward compatible)
|
| 146 |
-
- `sql_query`: Valid SQLite SELECT query (only present when route includes sql)
|
| 147 |
-
|
| 148 |
-
### Routing rules
|
| 149 |
-
|
| 150 |
-
| Query type | Route | Example |
|
| 151 |
-
|---|---|---|
|
| 152 |
-
| Specific data lookup | `sql` | "PTS scores for China along the years" |
|
| 153 |
-
| Aggregation / summary | `sql` | "How many countries are in the dataset?" |
|
| 154 |
-
| Data comparison | `sql` | "Compare GDP of China and India in 2020" |
|
| 155 |
-
| Conceptual / definitional | `vector` | "What does PTS measure?" |
|
| 156 |
-
| Mixed (concept + data) | `both` | "Explain PTS methodology and show China's scores" |
|
| 157 |
-
|
| 158 |
-
The QU prompt explicitly instructs: "If the user asks about data summarization, counting, averaging, filtering, or any question that requires looking at dataset rows, always set route to `sql` or `both`."
|
| 159 |
-
|
| 160 |
-
## SQL Retriever
|
| 161 |
-
|
| 162 |
-
New module `src/sql_retriever.py`:
|
| 163 |
-
|
| 164 |
-
### `execute_sql_query(sql_query, cfg) -> list[dict]`
|
| 165 |
-
|
| 166 |
-
- Opens `sql_db/knowledge_base.db` in **read-only mode**: `sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)`
|
| 167 |
-
- **SQL injection protection (two layers):**
|
| 168 |
-
1. Validate: strip whitespace, reject if contains `;`, reject if doesn't start with `SELECT` (case-insensitive)
|
| 169 |
-
2. Read-only SQLite connection (belt and suspenders)
|
| 170 |
-
- Executes with row limit (default 200) to prevent overwhelming LLM context
|
| 171 |
-
- Returns list of row dicts: `[{"Country": "China", "Year": 2005, "PTS_A": 4.0}, ...]`
|
| 172 |
-
|
| 173 |
-
### `format_sql_results_as_context(rows, sql_query, source_file) -> str`
|
| 174 |
-
|
| 175 |
-
```
|
| 176 |
-
=== SQL Query Results (PRIMARY — from local dataset) ===
|
| 177 |
-
|
| 178 |
-
Query: SELECT Country, Year, PTS_A FROM pts_dataset__pts_data_csv WHERE Country = 'China' ORDER BY Year
|
| 179 |
-
Source: PTS_dataset/pts_data.csv
|
| 180 |
-
Rows returned: 24
|
| 181 |
-
|
| 182 |
-
[CHUNK-SQL-001] Country = China, Year = 2000, PTS_A = 4.0
|
| 183 |
-
[CHUNK-SQL-002] Country = China, Year = 2001, PTS_A = 4.0
|
| 184 |
-
[CHUNK-SQL-003] Country = China, Year = 2002, PTS_A = 3.5
|
| 185 |
-
...
|
| 186 |
-
```
|
| 187 |
-
|
| 188 |
-
- Uses `[CHUNK-SQL-NNN]` chunk IDs for internal anchoring (consistent with `[CHUNK-LOCAL-NNN]` / `[CHUNK-WEB-NNN]` pattern — never shown to user)
|
| 189 |
-
- References the original source file in citations, not the SQL table name
|
| 190 |
-
|
| 191 |
-
### Integration in `src/retriever.py`
|
| 192 |
-
|
| 193 |
-
The `retrieve()` function gains `route` and `sql_query` parameters:
|
| 194 |
-
|
| 195 |
-
```python
|
| 196 |
-
def retrieve(query: str, cfg: dict, route: str = "vector", sql_query: str = None) -> dict:
|
| 197 |
-
```
|
| 198 |
-
|
| 199 |
-
- Routes to `retrieve_from_sql()`, `retrieve_from_vectordb()`, or both
|
| 200 |
-
- Fallback: if chosen path returns empty, tries the other
|
| 201 |
-
- SQL results merged into `build_combined_context()` as a third source type
|
| 202 |
-
|
| 203 |
-
## Prompt Changes
|
| 204 |
-
|
| 205 |
-
### System prompt (`SYSTEM_PROMPT_TEMPLATE`)
|
| 206 |
-
|
| 207 |
-
Update "Do NOT use chunk IDs" instruction:
|
| 208 |
-
```
|
| 209 |
-
Do NOT use chunk IDs (CHUNK-LOCAL-001, CHUNK-SQL-001, CHUNK-WEB-001, etc.)
|
| 210 |
-
```
|
| 211 |
-
|
| 212 |
-
### Verification prompt
|
| 213 |
-
|
| 214 |
-
Update check #6:
|
| 215 |
-
```
|
| 216 |
-
6. Is the source priority respected (CHUNK-LOCAL = CHUNK-SQL > CHUNK-WEB)?
|
| 217 |
-
```
|
| 218 |
-
|
| 219 |
-
SQL results from local files have equal priority with CHUNK-LOCAL (both are local data).
|
| 220 |
-
|
| 221 |
-
## Config Changes
|
| 222 |
-
|
| 223 |
-
### New config fields
|
| 224 |
-
|
| 225 |
-
| Config Path | Read By | Default | Notes |
|
| 226 |
-
|---|---|---|---|
|
| 227 |
-
| `paths.sql_db` | `src/sql_ingest.py`, `src/sql_retriever.py` | `"sql_db"` | Directory for SQLite DB + schema registry |
|
| 228 |
-
| `sql.enabled` | `src/retriever.py`, `src/query_engine.py` | `true` | Can disable if no tabular data |
|
| 229 |
-
| `sql.max_rows` | `src/sql_retriever.py` | `200` | Cap on SQL result rows sent to LLM |
|
| 230 |
-
|
| 231 |
-
### `.gitignore`
|
| 232 |
-
|
| 233 |
-
Add `sql_db/` alongside existing `chroma_db/`.
|
| 234 |
-
|
| 235 |
-
## File Changes Summary
|
| 236 |
-
|
| 237 |
-
### New files
|
| 238 |
-
- `src/sql_retriever.py` — SQL execution + result formatting
|
| 239 |
-
- `src/sql_ingest.py` — Tabular file -> SQLite loader + schema registry generation
|
| 240 |
-
- `sql_db/` — Directory for `knowledge_base.db` + `sql_schemas.json` (gitignored)
|
| 241 |
-
- `tests/test_sql_retriever.py` — Tests for SQL execution, validation, formatting
|
| 242 |
-
- `tests/test_sql_ingest.py` — Tests for SQLite ingestion, type inference, schema registry
|
| 243 |
-
|
| 244 |
-
### Modified files
|
| 245 |
-
- `src/ingest.py` — Call `sql_ingest` for tabular files during ingestion
|
| 246 |
-
- `src/retriever.py` — Add `route`/`sql_query` params, integrate SQL retrieval, merge results
|
| 247 |
-
- `src/query_engine.py` — Load schema, inject into QU prompt, parse `route`/`sql_query` from LLM output
|
| 248 |
-
- `src/prompts.py` — Extended QU prompt with schema + routing instructions; update system/verification prompts for CHUNK-SQL
|
| 249 |
-
- `setup.py` — Add `paths.sql_db`, `sql.enabled`, `sql.max_rows` to generated config
|
| 250 |
-
- `app_cli.py` — Pass `route`/`sql_query` from QU result through to retriever
|
| 251 |
-
- `app_web.py` — Same as CLI
|
| 252 |
-
- `CLAUDE.md` — Document new config fields and SQL architecture
|
| 253 |
-
- `README.md` — Document SQL layer feature
|
| 254 |
-
- `.gitignore` — Add `sql_db/`
|
| 255 |
-
|
| 256 |
-
### Unchanged
|
| 257 |
-
- `src/verifier.py` — SQL results are just context; 6-layer stack unchanged
|
| 258 |
-
- `src/llm/` — No changes
|
| 259 |
-
- `src/search/` — No changes
|
| 260 |
-
- `src/readers/` — No changes (still produce text for ChromaDB)
|
| 261 |
-
|
| 262 |
-
## Graceful Degradation
|
| 263 |
-
|
| 264 |
-
The SQL layer degrades gracefully at every level:
|
| 265 |
-
|
| 266 |
-
| Failure | Behavior |
|
| 267 |
-
|---|---|
|
| 268 |
-
| No tabular files ingested | No `sql_schemas.json` -> QU skips schema injection -> always routes to vector -> identical to current system |
|
| 269 |
-
| `sql.enabled: false` | QU skips schema injection -> always routes to vector |
|
| 270 |
-
| LLM omits `route` field | Defaults to `"vector"` (backward compatible) |
|
| 271 |
-
| LLM generates malformed SQL | Execution fails -> returns empty -> fallback to vector search |
|
| 272 |
-
| SQL query returns 0 rows | Fallback to vector search |
|
| 273 |
-
| Vector search returns 0 results | Fallback to SQL (if schemas exist) |
|
| 274 |
-
| `sql_schemas.json` corrupted | Caught, logged, QU skips schema injection |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/plans/2026-03-05-sql-layer-plan.md
DELETED
|
@@ -1,1719 +0,0 @@
|
|
| 1 |
-
# SQL Layer for Tabular Datasets — Implementation Plan
|
| 2 |
-
|
| 3 |
-
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
| 4 |
-
|
| 5 |
-
**Goal:** Add SQLite-based structured query retrieval alongside existing vector search so tabular data queries (filtering, aggregation, lookups) work correctly.
|
| 6 |
-
|
| 7 |
-
**Architecture:** Tabular files get dual-ingested into both ChromaDB (text chunks) and SQLite (structured tables). The query understanding layer routes queries to SQL, vector, or both paths with fallback. SQL results are formatted as context and fed through the unchanged 6-layer verification pipeline.
|
| 8 |
-
|
| 9 |
-
**Tech Stack:** Python 3.11+, SQLite (stdlib), existing ChromaDB/LLM stack
|
| 10 |
-
|
| 11 |
-
**Design doc:** `docs/plans/2026-03-05-sql-layer-design.md`
|
| 12 |
-
|
| 13 |
-
---
|
| 14 |
-
|
| 15 |
-
### Task 1: SQL Ingestion — Helpers (table naming, type inference, samples)
|
| 16 |
-
|
| 17 |
-
**Files:**
|
| 18 |
-
- Create: `src/sql_ingest.py`
|
| 19 |
-
- Create: `tests/test_sql_ingest.py`
|
| 20 |
-
|
| 21 |
-
**Step 1: Write the failing tests**
|
| 22 |
-
|
| 23 |
-
```python
|
| 24 |
-
# tests/test_sql_ingest.py
|
| 25 |
-
import pytest
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
def test_sanitize_table_name_basic():
|
| 29 |
-
from src.sql_ingest import _sanitize_table_name
|
| 30 |
-
assert _sanitize_table_name("PTS_dataset", "pts_data", ".csv") == "pts_dataset__pts_data_csv"
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def test_sanitize_table_name_with_sheet():
|
| 34 |
-
from src.sql_ingest import _sanitize_table_name
|
| 35 |
-
result = _sanitize_table_name("econ", "gdp", ".xlsx", "Sheet1")
|
| 36 |
-
assert result == "econ__gdp_xlsx_sheet1"
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
def test_sanitize_table_name_special_chars():
|
| 40 |
-
from src.sql_ingest import _sanitize_table_name
|
| 41 |
-
result = _sanitize_table_name("my-data", "file (2)", ".csv")
|
| 42 |
-
assert result == "my_data__file_2_csv"
|
| 43 |
-
assert result[0].isalpha()
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def test_sanitize_table_name_starts_with_number():
|
| 47 |
-
from src.sql_ingest import _sanitize_table_name
|
| 48 |
-
result = _sanitize_table_name("123data", "file", ".csv")
|
| 49 |
-
assert result[0].isalpha()
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
def test_infer_column_type_integer():
|
| 53 |
-
from src.sql_ingest import _infer_column_type
|
| 54 |
-
assert _infer_column_type(["1", "2", "3", None, ""]) == "INTEGER"
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
def test_infer_column_type_real():
|
| 58 |
-
from src.sql_ingest import _infer_column_type
|
| 59 |
-
assert _infer_column_type(["1.5", "2.0", "3.7"]) == "REAL"
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
def test_infer_column_type_text():
|
| 63 |
-
from src.sql_ingest import _infer_column_type
|
| 64 |
-
assert _infer_column_type(["China", "India", "Brazil"]) == "TEXT"
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
def test_infer_column_type_mixed_defaults_text():
|
| 68 |
-
from src.sql_ingest import _infer_column_type
|
| 69 |
-
assert _infer_column_type(["1", "two", "3"]) == "TEXT"
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
def test_infer_column_type_empty():
|
| 73 |
-
from src.sql_ingest import _infer_column_type
|
| 74 |
-
assert _infer_column_type([None, "", "nan"]) == "TEXT"
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def test_get_sample_values():
|
| 78 |
-
from src.sql_ingest import _get_sample_values
|
| 79 |
-
samples = _get_sample_values(["China", "India", "China", "Brazil", None, ""], n=3)
|
| 80 |
-
assert samples == ["China", "India", "Brazil"]
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def test_get_sample_values_fewer_than_n():
|
| 84 |
-
from src.sql_ingest import _get_sample_values
|
| 85 |
-
samples = _get_sample_values(["a", None, "a"], n=3)
|
| 86 |
-
assert samples == ["a"]
|
| 87 |
-
```
|
| 88 |
-
|
| 89 |
-
**Step 2: Run tests to verify they fail**
|
| 90 |
-
|
| 91 |
-
Run: `python -m pytest tests/test_sql_ingest.py -v`
|
| 92 |
-
Expected: FAIL — `ModuleNotFoundError: No module named 'src.sql_ingest'`
|
| 93 |
-
|
| 94 |
-
**Step 3: Write minimal implementation**
|
| 95 |
-
|
| 96 |
-
```python
|
| 97 |
-
# src/sql_ingest.py
|
| 98 |
-
"""SQL ingestion: load tabular files into SQLite for structured queries."""
|
| 99 |
-
|
| 100 |
-
import re
|
| 101 |
-
|
| 102 |
-
# Extensions that trigger SQL ingestion (tabular formats)
|
| 103 |
-
SQL_EXTENSIONS = {".csv", ".tab", ".tsv", ".xlsx", ".xls", ".dta", ".sav", ".rds", ".rda"}
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
def _sanitize_table_name(dataset: str, stem: str, ext: str, sheet: str = None) -> str:
|
| 107 |
-
"""Convert dataset/filename/ext into a valid SQL table identifier."""
|
| 108 |
-
name = f"{dataset}__{stem}_{ext.lstrip('.')}"
|
| 109 |
-
if sheet:
|
| 110 |
-
name += f"_{sheet}"
|
| 111 |
-
name = re.sub(r"[^a-zA-Z0-9]", "_", name)
|
| 112 |
-
name = re.sub(r"_+", "_", name).strip("_").lower()
|
| 113 |
-
if not name or not name[0].isalpha():
|
| 114 |
-
name = "t_" + name
|
| 115 |
-
return name
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
def _infer_column_type(values: list) -> str:
|
| 119 |
-
"""Infer SQLite column type from a list of raw string values."""
|
| 120 |
-
non_empty = [v for v in values if v is not None and str(v).strip() and str(v).lower() != "nan"]
|
| 121 |
-
if not non_empty:
|
| 122 |
-
return "TEXT"
|
| 123 |
-
# Try INTEGER
|
| 124 |
-
try:
|
| 125 |
-
for v in non_empty:
|
| 126 |
-
int(str(v).strip())
|
| 127 |
-
return "INTEGER"
|
| 128 |
-
except (ValueError, TypeError):
|
| 129 |
-
pass
|
| 130 |
-
# Try REAL
|
| 131 |
-
try:
|
| 132 |
-
for v in non_empty:
|
| 133 |
-
float(str(v).strip())
|
| 134 |
-
return "REAL"
|
| 135 |
-
except (ValueError, TypeError):
|
| 136 |
-
pass
|
| 137 |
-
return "TEXT"
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
def _get_sample_values(values: list, n: int = 3) -> list:
|
| 141 |
-
"""Get up to n unique non-empty sample values."""
|
| 142 |
-
seen = set()
|
| 143 |
-
samples = []
|
| 144 |
-
for v in values:
|
| 145 |
-
if v is not None and str(v).strip() and str(v).lower() != "nan":
|
| 146 |
-
v_str = str(v).strip()
|
| 147 |
-
if v_str not in seen:
|
| 148 |
-
seen.add(v_str)
|
| 149 |
-
samples.append(v_str)
|
| 150 |
-
if len(samples) >= n:
|
| 151 |
-
break
|
| 152 |
-
return samples
|
| 153 |
-
```
|
| 154 |
-
|
| 155 |
-
**Step 4: Run tests to verify they pass**
|
| 156 |
-
|
| 157 |
-
Run: `python -m pytest tests/test_sql_ingest.py -v`
|
| 158 |
-
Expected: 11 PASSED
|
| 159 |
-
|
| 160 |
-
**Step 5: Commit**
|
| 161 |
-
|
| 162 |
-
```bash
|
| 163 |
-
git add src/sql_ingest.py tests/test_sql_ingest.py
|
| 164 |
-
git commit -m "feat: add SQL ingestion helpers — table naming, type inference, samples"
|
| 165 |
-
```
|
| 166 |
-
|
| 167 |
-
---
|
| 168 |
-
|
| 169 |
-
### Task 2: SQL Ingestion — Core ingestion function
|
| 170 |
-
|
| 171 |
-
**Files:**
|
| 172 |
-
- Modify: `src/sql_ingest.py`
|
| 173 |
-
- Modify: `tests/test_sql_ingest.py`
|
| 174 |
-
|
| 175 |
-
**Step 1: Write the failing tests**
|
| 176 |
-
|
| 177 |
-
Append to `tests/test_sql_ingest.py`:
|
| 178 |
-
|
| 179 |
-
```python
|
| 180 |
-
import csv
|
| 181 |
-
import json
|
| 182 |
-
import sqlite3
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
def test_ingest_to_sql_creates_db_and_schema(tmp_path):
|
| 186 |
-
from pathlib import Path
|
| 187 |
-
from src.sql_ingest import ingest_to_sql
|
| 188 |
-
|
| 189 |
-
# Create a CSV file
|
| 190 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 191 |
-
ds_dir = kb_dir / "testds"
|
| 192 |
-
ds_dir.mkdir(parents=True)
|
| 193 |
-
csv_file = ds_dir / "data.csv"
|
| 194 |
-
csv_file.write_text("Country,Year,Score\nChina,2005,4.0\nIndia,2005,3.0\n")
|
| 195 |
-
|
| 196 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 197 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 198 |
-
files = [(Path(csv_file), "testds")]
|
| 199 |
-
|
| 200 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 201 |
-
|
| 202 |
-
# Check DB was created
|
| 203 |
-
db_path = sql_db_dir / "knowledge_base.db"
|
| 204 |
-
assert db_path.exists()
|
| 205 |
-
|
| 206 |
-
# Check schema registry
|
| 207 |
-
schema_path = sql_db_dir / "sql_schemas.json"
|
| 208 |
-
assert schema_path.exists()
|
| 209 |
-
with open(schema_path) as f:
|
| 210 |
-
saved = json.load(f)
|
| 211 |
-
assert len(saved) == 1
|
| 212 |
-
table_name = list(saved.keys())[0]
|
| 213 |
-
assert "testds" in table_name
|
| 214 |
-
assert saved[table_name]["row_count"] == 2
|
| 215 |
-
assert len(saved[table_name]["columns"]) == 3
|
| 216 |
-
|
| 217 |
-
# Check data in SQLite
|
| 218 |
-
conn = sqlite3.connect(str(db_path))
|
| 219 |
-
rows = conn.execute(f'SELECT * FROM "{table_name}"').fetchall()
|
| 220 |
-
conn.close()
|
| 221 |
-
assert len(rows) == 2
|
| 222 |
-
assert rows[0][0] == "China"
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
def test_ingest_to_sql_type_inference(tmp_path):
|
| 226 |
-
from pathlib import Path
|
| 227 |
-
from src.sql_ingest import ingest_to_sql
|
| 228 |
-
|
| 229 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 230 |
-
ds_dir = kb_dir / "ds"
|
| 231 |
-
ds_dir.mkdir(parents=True)
|
| 232 |
-
csv_file = ds_dir / "typed.csv"
|
| 233 |
-
csv_file.write_text("Name,Year,Score\nChina,2005,4.5\nIndia,2006,3.0\n")
|
| 234 |
-
|
| 235 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 236 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 237 |
-
files = [(Path(csv_file), "ds")]
|
| 238 |
-
|
| 239 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 240 |
-
table_name = list(schema.keys())[0]
|
| 241 |
-
|
| 242 |
-
# Name=TEXT, Year=INTEGER, Score=REAL
|
| 243 |
-
col_types = {c["name"]: c["type"] for c in schema[table_name]["columns"]}
|
| 244 |
-
assert col_types["Name"] == "TEXT"
|
| 245 |
-
assert col_types["Year"] == "INTEGER"
|
| 246 |
-
assert col_types["Score"] == "REAL"
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
def test_ingest_to_sql_no_tabular_files(tmp_path):
|
| 250 |
-
from pathlib import Path
|
| 251 |
-
from src.sql_ingest import ingest_to_sql
|
| 252 |
-
|
| 253 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 254 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 255 |
-
# Pass empty list (no tabular files)
|
| 256 |
-
schema = ingest_to_sql([], str(tmp_path), cfg)
|
| 257 |
-
assert schema == {}
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
def test_ingest_to_sql_clears_on_rerun(tmp_path):
|
| 261 |
-
from pathlib import Path
|
| 262 |
-
from src.sql_ingest import ingest_to_sql
|
| 263 |
-
|
| 264 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 265 |
-
ds_dir = kb_dir / "ds"
|
| 266 |
-
ds_dir.mkdir(parents=True)
|
| 267 |
-
csv_file = ds_dir / "data.csv"
|
| 268 |
-
csv_file.write_text("A,B\n1,2\n3,4\n")
|
| 269 |
-
|
| 270 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 271 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 272 |
-
files = [(Path(csv_file), "ds")]
|
| 273 |
-
|
| 274 |
-
# First run
|
| 275 |
-
ingest_to_sql(files, str(kb_dir), cfg)
|
| 276 |
-
# Second run — should not duplicate
|
| 277 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 278 |
-
table_name = list(schema.keys())[0]
|
| 279 |
-
assert schema[table_name]["row_count"] == 2
|
| 280 |
-
```
|
| 281 |
-
|
| 282 |
-
**Step 2: Run tests to verify they fail**
|
| 283 |
-
|
| 284 |
-
Run: `python -m pytest tests/test_sql_ingest.py::test_ingest_to_sql_creates_db_and_schema -v`
|
| 285 |
-
Expected: FAIL — `ImportError: cannot import name 'ingest_to_sql'`
|
| 286 |
-
|
| 287 |
-
**Step 3: Write the implementation**
|
| 288 |
-
|
| 289 |
-
Add to `src/sql_ingest.py`:
|
| 290 |
-
|
| 291 |
-
```python
|
| 292 |
-
import csv
|
| 293 |
-
import json
|
| 294 |
-
import os
|
| 295 |
-
import sqlite3
|
| 296 |
-
from pathlib import Path
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
def _load_rows_from_csv(file_path: str, ext: str) -> list[tuple]:
|
| 300 |
-
"""Load CSV/TSV/TAB into (sheet_or_name, headers, rows) tuples."""
|
| 301 |
-
delimiter = "\t" if ext in (".tab", ".tsv") else ","
|
| 302 |
-
with open(file_path, "r", encoding="utf-8", errors="replace") as f:
|
| 303 |
-
reader = csv.reader(f, delimiter=delimiter)
|
| 304 |
-
all_rows = list(reader)
|
| 305 |
-
if len(all_rows) < 2:
|
| 306 |
-
return []
|
| 307 |
-
headers = [h.strip().strip('"') for h in all_rows[0]]
|
| 308 |
-
return [(None, headers, all_rows[1:])]
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
def _load_rows_from_excel(file_path: str, ext: str) -> list[tuple]:
|
| 312 |
-
"""Load Excel into (sheet_name, headers, rows) tuples."""
|
| 313 |
-
if ext == ".xlsx":
|
| 314 |
-
import openpyxl
|
| 315 |
-
wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)
|
| 316 |
-
results = []
|
| 317 |
-
for sheet_name in wb.sheetnames:
|
| 318 |
-
ws = wb[sheet_name]
|
| 319 |
-
all_rows = list(ws.iter_rows(values_only=True))
|
| 320 |
-
if len(all_rows) < 2:
|
| 321 |
-
continue
|
| 322 |
-
headers = [str(h) if h is not None else "" for h in all_rows[0]]
|
| 323 |
-
rows = [[str(c) if c is not None else "" for c in row] for row in all_rows[1:]]
|
| 324 |
-
results.append((sheet_name, headers, rows))
|
| 325 |
-
wb.close()
|
| 326 |
-
return results
|
| 327 |
-
else: # .xls
|
| 328 |
-
import xlrd
|
| 329 |
-
wb = xlrd.open_workbook(file_path)
|
| 330 |
-
results = []
|
| 331 |
-
for sheet_idx in range(wb.nsheets):
|
| 332 |
-
ws = wb.sheet_by_index(sheet_idx)
|
| 333 |
-
if ws.nrows < 2:
|
| 334 |
-
continue
|
| 335 |
-
headers = [str(ws.cell_value(0, c)) for c in range(ws.ncols)]
|
| 336 |
-
rows = [[str(ws.cell_value(r, c)) for c in range(ws.ncols)] for r in range(1, ws.nrows)]
|
| 337 |
-
results.append((ws.name, headers, rows))
|
| 338 |
-
return results
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
def _load_rows_from_stata(file_path: str) -> list[tuple]:
|
| 342 |
-
"""Load Stata .dta into (None, headers, rows) tuples."""
|
| 343 |
-
import pyreadstat
|
| 344 |
-
df, _meta = pyreadstat.read_dta(file_path)
|
| 345 |
-
headers = list(df.columns)
|
| 346 |
-
rows = [[str(v) for v in row] for row in df.values.tolist()]
|
| 347 |
-
return [(None, headers, rows)]
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
def _load_rows_from_spss(file_path: str) -> list[tuple]:
|
| 351 |
-
"""Load SPSS .sav into (None, headers, rows) tuples."""
|
| 352 |
-
import pyreadstat
|
| 353 |
-
df, _meta = pyreadstat.read_sav(file_path)
|
| 354 |
-
headers = list(df.columns)
|
| 355 |
-
rows = [[str(v) for v in row] for row in df.values.tolist()]
|
| 356 |
-
return [(None, headers, rows)]
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
def _load_rows_from_rdata(file_path: str) -> list[tuple]:
|
| 360 |
-
"""Load R data files into (object_name, headers, rows) tuples."""
|
| 361 |
-
import pyreadr
|
| 362 |
-
result = pyreadr.read_r(file_path)
|
| 363 |
-
tables = []
|
| 364 |
-
for name, df in result.items():
|
| 365 |
-
headers = list(df.columns)
|
| 366 |
-
rows = [[str(v) for v in row] for row in df.values.tolist()]
|
| 367 |
-
tables.append((name, headers, rows))
|
| 368 |
-
return tables
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
def _load_tabular_file(file_path: str, ext: str) -> list[tuple]:
|
| 372 |
-
"""Dispatch to the correct loader based on extension.
|
| 373 |
-
|
| 374 |
-
Returns list of (sheet_or_name, headers, rows) tuples.
|
| 375 |
-
"""
|
| 376 |
-
if ext in (".csv", ".tab", ".tsv"):
|
| 377 |
-
return _load_rows_from_csv(file_path, ext)
|
| 378 |
-
if ext in (".xlsx", ".xls"):
|
| 379 |
-
return _load_rows_from_excel(file_path, ext)
|
| 380 |
-
if ext == ".dta":
|
| 381 |
-
return _load_rows_from_stata(file_path)
|
| 382 |
-
if ext == ".sav":
|
| 383 |
-
return _load_rows_from_spss(file_path)
|
| 384 |
-
if ext in (".rds", ".rda"):
|
| 385 |
-
return _load_rows_from_rdata(file_path)
|
| 386 |
-
return []
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
def _sanitize_column_name(name: str) -> str:
|
| 390 |
-
"""Sanitize a column name into a valid SQL identifier."""
|
| 391 |
-
safe = re.sub(r"[^a-zA-Z0-9_]", "_", name).strip("_")
|
| 392 |
-
if not safe or not safe[0].isalpha():
|
| 393 |
-
safe = "col_" + safe
|
| 394 |
-
return safe
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
def ingest_to_sql(files: list[tuple], documents_dir: str, cfg: dict) -> dict:
|
| 398 |
-
"""Ingest tabular files into SQLite and generate schema registry.
|
| 399 |
-
|
| 400 |
-
Args:
|
| 401 |
-
files: List of (file_path, dataset_name) tuples.
|
| 402 |
-
documents_dir: Root documents directory.
|
| 403 |
-
cfg: App config dict.
|
| 404 |
-
|
| 405 |
-
Returns:
|
| 406 |
-
Schema registry dict (also saved to sql_schemas.json).
|
| 407 |
-
"""
|
| 408 |
-
sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
|
| 409 |
-
if not os.path.isabs(sql_db_dir):
|
| 410 |
-
project_root = Path(__file__).resolve().parent.parent
|
| 411 |
-
sql_db_dir = os.path.join(str(project_root), sql_db_dir)
|
| 412 |
-
|
| 413 |
-
os.makedirs(sql_db_dir, exist_ok=True)
|
| 414 |
-
db_path = os.path.join(sql_db_dir, "knowledge_base.db")
|
| 415 |
-
schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
|
| 416 |
-
|
| 417 |
-
# Filter to tabular files only
|
| 418 |
-
tabular_files = [(fp, ds) for fp, ds in files if fp.suffix.lower() in SQL_EXTENSIONS]
|
| 419 |
-
|
| 420 |
-
if not tabular_files:
|
| 421 |
-
if os.path.exists(schema_path):
|
| 422 |
-
os.remove(schema_path)
|
| 423 |
-
return {}
|
| 424 |
-
|
| 425 |
-
# Clear existing DB
|
| 426 |
-
if os.path.exists(db_path):
|
| 427 |
-
os.remove(db_path)
|
| 428 |
-
|
| 429 |
-
conn = sqlite3.connect(db_path)
|
| 430 |
-
schema_registry = {}
|
| 431 |
-
|
| 432 |
-
for file_path, dataset_name in tabular_files:
|
| 433 |
-
ext = file_path.suffix.lower()
|
| 434 |
-
rel_path = file_path.relative_to(documents_dir)
|
| 435 |
-
source_file = str(rel_path)
|
| 436 |
-
|
| 437 |
-
try:
|
| 438 |
-
tables = _load_tabular_file(str(file_path), ext)
|
| 439 |
-
except Exception as e:
|
| 440 |
-
print(f" SQL ingest error for {file_path.name}: {e}")
|
| 441 |
-
continue
|
| 442 |
-
|
| 443 |
-
for sheet_or_name, headers, rows in tables:
|
| 444 |
-
table_name = _sanitize_table_name(dataset_name, file_path.stem, ext, sheet_or_name)
|
| 445 |
-
|
| 446 |
-
# Infer column types and collect samples
|
| 447 |
-
safe_headers = [_sanitize_column_name(h) for h in headers]
|
| 448 |
-
col_types = []
|
| 449 |
-
col_samples = []
|
| 450 |
-
for col_idx in range(len(headers)):
|
| 451 |
-
col_values = [row[col_idx] if col_idx < len(row) else None for row in rows]
|
| 452 |
-
col_types.append(_infer_column_type(col_values))
|
| 453 |
-
col_samples.append(_get_sample_values(col_values))
|
| 454 |
-
|
| 455 |
-
# Create table
|
| 456 |
-
col_defs = ", ".join(f'"{h}" {t}' for h, t in zip(safe_headers, col_types))
|
| 457 |
-
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
| 458 |
-
conn.execute(f'CREATE TABLE "{table_name}" ({col_defs})')
|
| 459 |
-
|
| 460 |
-
# Insert rows
|
| 461 |
-
placeholders = ", ".join(["?"] * len(safe_headers))
|
| 462 |
-
insert_sql = f'INSERT INTO "{table_name}" VALUES ({placeholders})'
|
| 463 |
-
|
| 464 |
-
for row in rows:
|
| 465 |
-
values = []
|
| 466 |
-
for col_idx, col_type in enumerate(col_types):
|
| 467 |
-
raw = row[col_idx] if col_idx < len(row) else None
|
| 468 |
-
if raw is None or str(raw).strip() == "" or str(raw).lower() == "nan":
|
| 469 |
-
values.append(None)
|
| 470 |
-
elif col_type == "INTEGER":
|
| 471 |
-
try:
|
| 472 |
-
values.append(int(float(str(raw).strip())))
|
| 473 |
-
except (ValueError, TypeError):
|
| 474 |
-
values.append(None)
|
| 475 |
-
elif col_type == "REAL":
|
| 476 |
-
try:
|
| 477 |
-
values.append(float(str(raw).strip()))
|
| 478 |
-
except (ValueError, TypeError):
|
| 479 |
-
values.append(None)
|
| 480 |
-
else:
|
| 481 |
-
values.append(str(raw).strip())
|
| 482 |
-
conn.execute(insert_sql, values)
|
| 483 |
-
|
| 484 |
-
conn.commit()
|
| 485 |
-
|
| 486 |
-
row_count = conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone()[0]
|
| 487 |
-
|
| 488 |
-
schema_registry[table_name] = {
|
| 489 |
-
"source_file": source_file,
|
| 490 |
-
"columns": [
|
| 491 |
-
{
|
| 492 |
-
"name": safe_headers[i],
|
| 493 |
-
"original_name": headers[i],
|
| 494 |
-
"type": col_types[i],
|
| 495 |
-
"sample": col_samples[i],
|
| 496 |
-
}
|
| 497 |
-
for i in range(len(headers))
|
| 498 |
-
],
|
| 499 |
-
"row_count": row_count,
|
| 500 |
-
}
|
| 501 |
-
print(f" SQL: {table_name} ({row_count} rows, {len(headers)} columns)")
|
| 502 |
-
|
| 503 |
-
conn.close()
|
| 504 |
-
|
| 505 |
-
with open(schema_path, "w", encoding="utf-8") as f:
|
| 506 |
-
json.dump(schema_registry, f, indent=2, default=str)
|
| 507 |
-
|
| 508 |
-
return schema_registry
|
| 509 |
-
```
|
| 510 |
-
|
| 511 |
-
**Step 4: Run tests to verify they pass**
|
| 512 |
-
|
| 513 |
-
Run: `python -m pytest tests/test_sql_ingest.py -v`
|
| 514 |
-
Expected: 15 PASSED
|
| 515 |
-
|
| 516 |
-
**Step 5: Commit**
|
| 517 |
-
|
| 518 |
-
```bash
|
| 519 |
-
git add src/sql_ingest.py tests/test_sql_ingest.py
|
| 520 |
-
git commit -m "feat: add SQL ingestion — load tabular files into SQLite with schema registry"
|
| 521 |
-
```
|
| 522 |
-
|
| 523 |
-
---
|
| 524 |
-
|
| 525 |
-
### Task 3: SQL Retriever — Validation and execution
|
| 526 |
-
|
| 527 |
-
**Files:**
|
| 528 |
-
- Create: `src/sql_retriever.py`
|
| 529 |
-
- Create: `tests/test_sql_retriever.py`
|
| 530 |
-
|
| 531 |
-
**Step 1: Write the failing tests**
|
| 532 |
-
|
| 533 |
-
```python
|
| 534 |
-
# tests/test_sql_retriever.py
|
| 535 |
-
import pytest
|
| 536 |
-
import json
|
| 537 |
-
import sqlite3
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
def test_validate_sql_rejects_non_select():
|
| 541 |
-
from src.sql_retriever import _validate_sql
|
| 542 |
-
assert _validate_sql("DROP TABLE users") is False
|
| 543 |
-
assert _validate_sql("INSERT INTO t VALUES (1)") is False
|
| 544 |
-
assert _validate_sql("UPDATE t SET x=1") is False
|
| 545 |
-
assert _validate_sql("DELETE FROM t") is False
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
def test_validate_sql_rejects_semicolons():
|
| 549 |
-
from src.sql_retriever import _validate_sql
|
| 550 |
-
assert _validate_sql("SELECT 1; DROP TABLE users") is False
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
def test_validate_sql_accepts_select():
|
| 554 |
-
from src.sql_retriever import _validate_sql
|
| 555 |
-
assert _validate_sql("SELECT * FROM t") is True
|
| 556 |
-
assert _validate_sql(" select Country, Year from t WHERE x = 1 ") is True
|
| 557 |
-
assert _validate_sql("SELECT COUNT(*) FROM t GROUP BY x") is True
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
def test_validate_sql_rejects_empty():
|
| 561 |
-
from src.sql_retriever import _validate_sql
|
| 562 |
-
assert _validate_sql("") is False
|
| 563 |
-
assert _validate_sql(" ") is False
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
def test_execute_sql_query_returns_rows(tmp_path):
|
| 567 |
-
from src.sql_retriever import execute_sql_query
|
| 568 |
-
|
| 569 |
-
# Create a test DB
|
| 570 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 571 |
-
db_path.parent.mkdir()
|
| 572 |
-
conn = sqlite3.connect(str(db_path))
|
| 573 |
-
conn.execute("CREATE TABLE t (Country TEXT, Year INTEGER, Score REAL)")
|
| 574 |
-
conn.execute("INSERT INTO t VALUES ('China', 2005, 4.0)")
|
| 575 |
-
conn.execute("INSERT INTO t VALUES ('India', 2005, 3.0)")
|
| 576 |
-
conn.commit()
|
| 577 |
-
conn.close()
|
| 578 |
-
|
| 579 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}, "sql": {"max_rows": 200}}
|
| 580 |
-
rows = execute_sql_query("SELECT * FROM t WHERE Country = 'China'", cfg)
|
| 581 |
-
assert len(rows) == 1
|
| 582 |
-
assert rows[0]["Country"] == "China"
|
| 583 |
-
assert rows[0]["Score"] == 4.0
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
def test_execute_sql_query_respects_max_rows(tmp_path):
|
| 587 |
-
from src.sql_retriever import execute_sql_query
|
| 588 |
-
|
| 589 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 590 |
-
db_path.parent.mkdir()
|
| 591 |
-
conn = sqlite3.connect(str(db_path))
|
| 592 |
-
conn.execute("CREATE TABLE t (x INTEGER)")
|
| 593 |
-
for i in range(100):
|
| 594 |
-
conn.execute("INSERT INTO t VALUES (?)", (i,))
|
| 595 |
-
conn.commit()
|
| 596 |
-
conn.close()
|
| 597 |
-
|
| 598 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}, "sql": {"max_rows": 5}}
|
| 599 |
-
rows = execute_sql_query("SELECT * FROM t", cfg)
|
| 600 |
-
assert len(rows) == 5
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
def test_execute_sql_query_rejects_non_select(tmp_path):
|
| 604 |
-
from src.sql_retriever import execute_sql_query
|
| 605 |
-
|
| 606 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 607 |
-
db_path.parent.mkdir()
|
| 608 |
-
conn = sqlite3.connect(str(db_path))
|
| 609 |
-
conn.execute("CREATE TABLE t (x INTEGER)")
|
| 610 |
-
conn.commit()
|
| 611 |
-
conn.close()
|
| 612 |
-
|
| 613 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}}
|
| 614 |
-
rows = execute_sql_query("DROP TABLE t", cfg)
|
| 615 |
-
assert rows == []
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
def test_execute_sql_query_handles_bad_sql(tmp_path):
|
| 619 |
-
from src.sql_retriever import execute_sql_query
|
| 620 |
-
|
| 621 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 622 |
-
db_path.parent.mkdir()
|
| 623 |
-
conn = sqlite3.connect(str(db_path))
|
| 624 |
-
conn.execute("CREATE TABLE t (x INTEGER)")
|
| 625 |
-
conn.commit()
|
| 626 |
-
conn.close()
|
| 627 |
-
|
| 628 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}}
|
| 629 |
-
rows = execute_sql_query("SELECT * FROM nonexistent_table", cfg)
|
| 630 |
-
assert rows == []
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
def test_execute_sql_query_no_db(tmp_path):
|
| 634 |
-
from src.sql_retriever import execute_sql_query
|
| 635 |
-
|
| 636 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db_missing")}}
|
| 637 |
-
rows = execute_sql_query("SELECT 1", cfg)
|
| 638 |
-
assert rows == []
|
| 639 |
-
```
|
| 640 |
-
|
| 641 |
-
**Step 2: Run tests to verify they fail**
|
| 642 |
-
|
| 643 |
-
Run: `python -m pytest tests/test_sql_retriever.py -v`
|
| 644 |
-
Expected: FAIL — `ModuleNotFoundError: No module named 'src.sql_retriever'`
|
| 645 |
-
|
| 646 |
-
**Step 3: Write the implementation**
|
| 647 |
-
|
| 648 |
-
```python
|
| 649 |
-
# src/sql_retriever.py
|
| 650 |
-
"""SQL retriever: execute validated queries against the SQLite knowledge base."""
|
| 651 |
-
|
| 652 |
-
import os
|
| 653 |
-
import sqlite3
|
| 654 |
-
from pathlib import Path
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
def _validate_sql(sql: str) -> bool:
|
| 658 |
-
"""Validate that a SQL string is a safe SELECT query.
|
| 659 |
-
|
| 660 |
-
Rejects non-SELECT statements and queries containing semicolons
|
| 661 |
-
(which could chain dangerous statements).
|
| 662 |
-
"""
|
| 663 |
-
stripped = sql.strip()
|
| 664 |
-
if not stripped:
|
| 665 |
-
return False
|
| 666 |
-
if ";" in stripped:
|
| 667 |
-
return False
|
| 668 |
-
if not stripped.upper().startswith("SELECT"):
|
| 669 |
-
return False
|
| 670 |
-
return True
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
def _get_db_path(cfg: dict) -> str:
|
| 674 |
-
"""Resolve the SQLite database path from config."""
|
| 675 |
-
sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
|
| 676 |
-
if not os.path.isabs(sql_db_dir):
|
| 677 |
-
project_root = Path(__file__).resolve().parent.parent
|
| 678 |
-
sql_db_dir = os.path.join(str(project_root), sql_db_dir)
|
| 679 |
-
return os.path.join(sql_db_dir, "knowledge_base.db")
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
def execute_sql_query(sql_query: str, cfg: dict) -> list[dict]:
|
| 683 |
-
"""Execute a validated SELECT query against the knowledge base SQLite DB.
|
| 684 |
-
|
| 685 |
-
Args:
|
| 686 |
-
sql_query: A SQL SELECT query (generated by the LLM).
|
| 687 |
-
cfg: App config dict.
|
| 688 |
-
|
| 689 |
-
Returns:
|
| 690 |
-
List of row dicts, or empty list on any error.
|
| 691 |
-
"""
|
| 692 |
-
if not _validate_sql(sql_query):
|
| 693 |
-
return []
|
| 694 |
-
|
| 695 |
-
db_path = _get_db_path(cfg)
|
| 696 |
-
if not os.path.exists(db_path):
|
| 697 |
-
return []
|
| 698 |
-
|
| 699 |
-
max_rows = cfg.get("sql", {}).get("max_rows", 200)
|
| 700 |
-
|
| 701 |
-
try:
|
| 702 |
-
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
| 703 |
-
conn.row_factory = sqlite3.Row
|
| 704 |
-
cursor = conn.execute(sql_query)
|
| 705 |
-
rows = cursor.fetchmany(max_rows)
|
| 706 |
-
result = [dict(row) for row in rows]
|
| 707 |
-
conn.close()
|
| 708 |
-
return result
|
| 709 |
-
except Exception:
|
| 710 |
-
return []
|
| 711 |
-
```
|
| 712 |
-
|
| 713 |
-
**Step 4: Run tests to verify they pass**
|
| 714 |
-
|
| 715 |
-
Run: `python -m pytest tests/test_sql_retriever.py -v`
|
| 716 |
-
Expected: 9 PASSED
|
| 717 |
-
|
| 718 |
-
**Step 5: Commit**
|
| 719 |
-
|
| 720 |
-
```bash
|
| 721 |
-
git add src/sql_retriever.py tests/test_sql_retriever.py
|
| 722 |
-
git commit -m "feat: add SQL retriever — validated read-only query execution"
|
| 723 |
-
```
|
| 724 |
-
|
| 725 |
-
---
|
| 726 |
-
|
| 727 |
-
### Task 4: SQL Retriever — Result formatting and schema summary
|
| 728 |
-
|
| 729 |
-
**Files:**
|
| 730 |
-
- Modify: `src/sql_retriever.py`
|
| 731 |
-
- Modify: `tests/test_sql_retriever.py`
|
| 732 |
-
|
| 733 |
-
**Step 1: Write the failing tests**
|
| 734 |
-
|
| 735 |
-
Append to `tests/test_sql_retriever.py`:
|
| 736 |
-
|
| 737 |
-
```python
|
| 738 |
-
def test_format_sql_results_empty():
|
| 739 |
-
from src.sql_retriever import format_sql_results_as_context
|
| 740 |
-
assert format_sql_results_as_context([], "SELECT 1", "data.csv") == ""
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
def test_format_sql_results_basic():
|
| 744 |
-
from src.sql_retriever import format_sql_results_as_context
|
| 745 |
-
rows = [
|
| 746 |
-
{"Country": "China", "Year": 2005, "PTS_A": 4.0},
|
| 747 |
-
{"Country": "China", "Year": 2006, "PTS_A": 3.5},
|
| 748 |
-
]
|
| 749 |
-
result = format_sql_results_as_context(rows, "SELECT * FROM t", "PTS_dataset/pts_data.csv")
|
| 750 |
-
assert "[CHUNK-SQL-001]" in result
|
| 751 |
-
assert "[CHUNK-SQL-002]" in result
|
| 752 |
-
assert "China" in result
|
| 753 |
-
assert "PTS_dataset/pts_data.csv" in result
|
| 754 |
-
assert "PRIMARY" in result
|
| 755 |
-
assert "SELECT * FROM t" in result
|
| 756 |
-
assert "Rows returned: 2" in result
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
def test_build_schema_summary_empty():
|
| 760 |
-
from src.sql_retriever import build_schema_summary
|
| 761 |
-
assert build_schema_summary({}) == ""
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
def test_build_schema_summary_basic():
|
| 765 |
-
from src.sql_retriever import build_schema_summary
|
| 766 |
-
schema = {
|
| 767 |
-
"test_table": {
|
| 768 |
-
"source_file": "data.csv",
|
| 769 |
-
"columns": [
|
| 770 |
-
{"name": "Country", "type": "TEXT", "sample": ["China", "India"]},
|
| 771 |
-
{"name": "Year", "type": "INTEGER", "sample": [2005, 2010]},
|
| 772 |
-
],
|
| 773 |
-
"row_count": 100,
|
| 774 |
-
}
|
| 775 |
-
}
|
| 776 |
-
result = build_schema_summary(schema)
|
| 777 |
-
assert "test_table" in result
|
| 778 |
-
assert "100 rows" in result
|
| 779 |
-
assert "Country (TEXT)" in result
|
| 780 |
-
assert "Year (INTEGER)" in result
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
def test_lookup_source_file():
|
| 784 |
-
from src.sql_retriever import _lookup_source_file
|
| 785 |
-
schema = {
|
| 786 |
-
"ds__data_csv": {"source_file": "ds/data.csv", "columns": [], "row_count": 0},
|
| 787 |
-
}
|
| 788 |
-
assert _lookup_source_file("ds__data_csv", schema) == "ds/data.csv"
|
| 789 |
-
assert _lookup_source_file("unknown", schema) == "unknown"
|
| 790 |
-
```
|
| 791 |
-
|
| 792 |
-
**Step 2: Run tests to verify they fail**
|
| 793 |
-
|
| 794 |
-
Run: `python -m pytest tests/test_sql_retriever.py::test_format_sql_results_basic -v`
|
| 795 |
-
Expected: FAIL — `ImportError: cannot import name 'format_sql_results_as_context'`
|
| 796 |
-
|
| 797 |
-
**Step 3: Add to `src/sql_retriever.py`**
|
| 798 |
-
|
| 799 |
-
```python
|
| 800 |
-
def format_sql_results_as_context(
|
| 801 |
-
rows: list[dict], sql_query: str, source_file: str,
|
| 802 |
-
) -> str:
|
| 803 |
-
"""Format SQL result rows as context for the verification pipeline.
|
| 804 |
-
|
| 805 |
-
Uses [CHUNK-SQL-NNN] IDs for internal anchoring, consistent with
|
| 806 |
-
CHUNK-LOCAL and CHUNK-WEB patterns.
|
| 807 |
-
"""
|
| 808 |
-
if not rows:
|
| 809 |
-
return ""
|
| 810 |
-
|
| 811 |
-
parts = [
|
| 812 |
-
"=== SQL Query Results (PRIMARY — from local dataset) ===\n",
|
| 813 |
-
f"Query: {sql_query}",
|
| 814 |
-
f"Source: {source_file}",
|
| 815 |
-
f"Rows returned: {len(rows)}\n",
|
| 816 |
-
]
|
| 817 |
-
for i, row in enumerate(rows, 1):
|
| 818 |
-
fields = ", ".join(f"{k} = {v}" for k, v in row.items())
|
| 819 |
-
parts.append(f"[CHUNK-SQL-{i:03d}] {fields}")
|
| 820 |
-
return "\n".join(parts)
|
| 821 |
-
|
| 822 |
-
|
| 823 |
-
def _lookup_source_file(table_name: str, schema: dict) -> str:
|
| 824 |
-
"""Look up the original source file for a SQL table name."""
|
| 825 |
-
entry = schema.get(table_name)
|
| 826 |
-
if entry:
|
| 827 |
-
return entry.get("source_file", table_name)
|
| 828 |
-
return table_name
|
| 829 |
-
|
| 830 |
-
|
| 831 |
-
def build_schema_summary(schema: dict) -> str:
|
| 832 |
-
"""Build a compact schema summary for injection into the QU prompt.
|
| 833 |
-
|
| 834 |
-
Format:
|
| 835 |
-
Available SQL tables:
|
| 836 |
-
- table_name (N rows): col1 (TYPE), col2 (TYPE), ...
|
| 837 |
-
"""
|
| 838 |
-
if not schema:
|
| 839 |
-
return ""
|
| 840 |
-
|
| 841 |
-
lines = ["Available SQL tables:"]
|
| 842 |
-
for table_name, info in schema.items():
|
| 843 |
-
cols = ", ".join(
|
| 844 |
-
f"{c['name']} ({c['type']})" for c in info.get("columns", [])
|
| 845 |
-
)
|
| 846 |
-
row_count = info.get("row_count", 0)
|
| 847 |
-
lines.append(f"- {table_name} ({row_count} rows): {cols}")
|
| 848 |
-
return "\n".join(lines)
|
| 849 |
-
```
|
| 850 |
-
|
| 851 |
-
**Step 4: Run tests to verify they pass**
|
| 852 |
-
|
| 853 |
-
Run: `python -m pytest tests/test_sql_retriever.py -v`
|
| 854 |
-
Expected: 14 PASSED
|
| 855 |
-
|
| 856 |
-
**Step 5: Commit**
|
| 857 |
-
|
| 858 |
-
```bash
|
| 859 |
-
git add src/sql_retriever.py tests/test_sql_retriever.py
|
| 860 |
-
git commit -m "feat: add SQL result formatting and schema summary builder"
|
| 861 |
-
```
|
| 862 |
-
|
| 863 |
-
---
|
| 864 |
-
|
| 865 |
-
### Task 5: Prompt updates — QU prompt + system + verification
|
| 866 |
-
|
| 867 |
-
**Files:**
|
| 868 |
-
- Modify: `src/prompts.py:124-164` (QU prompt template)
|
| 869 |
-
- Modify: `src/prompts.py:43-44` (system prompt — chunk ID line)
|
| 870 |
-
- Modify: `src/prompts.py:106` (verification prompt — check #6)
|
| 871 |
-
- Modify: `tests/test_prompts.py` (if it exists; otherwise create)
|
| 872 |
-
|
| 873 |
-
**Context:** Read `src/prompts.py` before editing. The QU prompt template uses `str.format()` with `{{` and `}}` for literal JSON braces.
|
| 874 |
-
|
| 875 |
-
**Step 1: Write the failing tests**
|
| 876 |
-
|
| 877 |
-
Create or append to `tests/test_prompts.py`:
|
| 878 |
-
|
| 879 |
-
```python
|
| 880 |
-
# tests/test_prompts.py
|
| 881 |
-
import pytest
|
| 882 |
-
|
| 883 |
-
|
| 884 |
-
def test_qu_prompt_includes_schema_when_provided():
|
| 885 |
-
from src.prompts import build_query_understanding_prompt
|
| 886 |
-
schema_summary = "Available SQL tables:\n- t (100 rows): x (TEXT)"
|
| 887 |
-
prompt = build_query_understanding_prompt(
|
| 888 |
-
"test query", "research", [], sql_schema_summary=schema_summary,
|
| 889 |
-
)
|
| 890 |
-
assert "Available SQL tables" in prompt
|
| 891 |
-
assert "route" in prompt
|
| 892 |
-
assert "sql_query" in prompt
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
def test_qu_prompt_no_schema_no_sql_fields():
|
| 896 |
-
from src.prompts import build_query_understanding_prompt
|
| 897 |
-
prompt = build_query_understanding_prompt("test query", "research", [])
|
| 898 |
-
assert "Available SQL tables" not in prompt
|
| 899 |
-
|
| 900 |
-
|
| 901 |
-
def test_system_prompt_mentions_chunk_sql():
|
| 902 |
-
from src.prompts import build_prompt
|
| 903 |
-
prompt = build_prompt("context", "Bot", "research")
|
| 904 |
-
assert "CHUNK-SQL" in prompt
|
| 905 |
-
|
| 906 |
-
|
| 907 |
-
def test_verification_prompt_sql_priority():
|
| 908 |
-
from src.prompts import build_verification_prompt
|
| 909 |
-
prompt = build_verification_prompt("resp", "ctx", [], [])
|
| 910 |
-
assert "CHUNK-SQL" in prompt
|
| 911 |
-
```
|
| 912 |
-
|
| 913 |
-
**Step 2: Run tests to verify they fail**
|
| 914 |
-
|
| 915 |
-
Run: `python -m pytest tests/test_prompts.py -v`
|
| 916 |
-
Expected: FAIL — `build_query_understanding_prompt() got an unexpected keyword argument 'sql_schema_summary'`
|
| 917 |
-
|
| 918 |
-
**Step 3: Edit `src/prompts.py`**
|
| 919 |
-
|
| 920 |
-
Three changes needed:
|
| 921 |
-
|
| 922 |
-
**3a.** Update `SYSTEM_PROMPT_TEMPLATE` (line ~43): Change the chunk ID line from:
|
| 923 |
-
```
|
| 924 |
-
Do NOT use chunk IDs (CHUNK-LOCAL-001, CHUNK-WEB-001, etc.)
|
| 925 |
-
```
|
| 926 |
-
to:
|
| 927 |
-
```
|
| 928 |
-
Do NOT use chunk IDs (CHUNK-LOCAL-001, CHUNK-SQL-001, CHUNK-WEB-001, etc.)
|
| 929 |
-
```
|
| 930 |
-
|
| 931 |
-
**3b.** Update `VERIFICATION_PROMPT_TEMPLATE` (line ~106): Change check #6 from:
|
| 932 |
-
```
|
| 933 |
-
6. Is the source priority respected (CHUNK-LOCAL > CHUNK-WEB)?
|
| 934 |
-
```
|
| 935 |
-
to:
|
| 936 |
-
```
|
| 937 |
-
6. Is the source priority respected (CHUNK-LOCAL = CHUNK-SQL > CHUNK-WEB)?
|
| 938 |
-
```
|
| 939 |
-
|
| 940 |
-
**3c.** Update `QUERY_UNDERSTANDING_PROMPT_TEMPLATE` — add SQL routing instructions and a `{sql_schema}` placeholder. Add this block after the CLARIFY rules, before the CONVERSATION HISTORY section:
|
| 941 |
-
|
| 942 |
-
```
|
| 943 |
-
{sql_routing_block}
|
| 944 |
-
```
|
| 945 |
-
|
| 946 |
-
Where `sql_routing_block` is either empty (no schemas) or:
|
| 947 |
-
|
| 948 |
-
```
|
| 949 |
-
SQL ROUTING:
|
| 950 |
-
You have access to structured SQL tables in addition to vector search.
|
| 951 |
-
When choosing a route:
|
| 952 |
-
- "sql" — for data lookups, filtering, aggregation, counting, comparisons
|
| 953 |
-
(e.g., "PTS scores for China", "how many countries", "average GDP")
|
| 954 |
-
- "vector" — for conceptual, definitional, or methodology questions
|
| 955 |
-
(e.g., "what does PTS measure?", "explain the methodology")
|
| 956 |
-
- "both" — for mixed questions combining concepts with data
|
| 957 |
-
(e.g., "explain PTS methodology and show China's scores")
|
| 958 |
-
|
| 959 |
-
IMPORTANT: If the user asks about data summarization, counting, averaging,
|
| 960 |
-
filtering, or any question that requires looking at dataset rows, ALWAYS
|
| 961 |
-
set route to "sql" or "both".
|
| 962 |
-
|
| 963 |
-
When route is "sql" or "both", also provide a valid SQLite SELECT query
|
| 964 |
-
in the "sql_query" field. Use only table/column names from the schema below.
|
| 965 |
-
|
| 966 |
-
{schema_text}
|
| 967 |
-
```
|
| 968 |
-
|
| 969 |
-
Also update the JSON output format in the template to include `route` and `sql_query`:
|
| 970 |
-
|
| 971 |
-
```
|
| 972 |
-
Respond in JSON:
|
| 973 |
-
{{
|
| 974 |
-
"action": "search" or "clarify",
|
| 975 |
-
"route": "vector" or "sql" or "both",
|
| 976 |
-
"search_query": "keyword-optimized query for vector search",
|
| 977 |
-
"display_query": "clear natural-language question for the AI to answer",
|
| 978 |
-
"sql_query": "SQLite SELECT query (only when route is sql or both)",
|
| 979 |
-
"clarification_question": "question to ask (only if action is clarify)",
|
| 980 |
-
"reasoning": "one sentence explaining your choice"
|
| 981 |
-
}}
|
| 982 |
-
```
|
| 983 |
-
|
| 984 |
-
**3d.** Update `build_query_understanding_prompt()` signature to accept `sql_schema_summary`:
|
| 985 |
-
|
| 986 |
-
```python
|
| 987 |
-
def build_query_understanding_prompt(
|
| 988 |
-
query: str, domain: str, history: list[dict],
|
| 989 |
-
sql_schema_summary: str = "",
|
| 990 |
-
) -> str:
|
| 991 |
-
```
|
| 992 |
-
|
| 993 |
-
Build the `sql_routing_block`: if `sql_schema_summary` is non-empty, format the full SQL routing instructions with schema. Otherwise use empty string. Pass it through `_escape_braces()`.
|
| 994 |
-
|
| 995 |
-
**Step 4: Run tests to verify they pass**
|
| 996 |
-
|
| 997 |
-
Run: `python -m pytest tests/test_prompts.py -v`
|
| 998 |
-
Expected: 4 PASSED
|
| 999 |
-
|
| 1000 |
-
Then run full suite to make sure nothing broke:
|
| 1001 |
-
|
| 1002 |
-
Run: `python -m pytest tests/ -v`
|
| 1003 |
-
Expected: All tests PASS
|
| 1004 |
-
|
| 1005 |
-
**Step 5: Commit**
|
| 1006 |
-
|
| 1007 |
-
```bash
|
| 1008 |
-
git add src/prompts.py tests/test_prompts.py
|
| 1009 |
-
git commit -m "feat: update prompts for SQL routing, CHUNK-SQL IDs, and schema injection"
|
| 1010 |
-
```
|
| 1011 |
-
|
| 1012 |
-
---
|
| 1013 |
-
|
| 1014 |
-
### Task 6: Query engine — Schema loading and route/sql_query parsing
|
| 1015 |
-
|
| 1016 |
-
**Files:**
|
| 1017 |
-
- Modify: `src/query_engine.py:10-72`
|
| 1018 |
-
- Modify: `tests/test_query_engine.py` (read first to see existing tests)
|
| 1019 |
-
|
| 1020 |
-
**Context:** Read `src/query_engine.py` and `tests/test_query_engine.py` before editing. The QU layer already parses `action`, `search_query`, `display_query`. We need to also parse `route` and `sql_query` from the LLM output, and load the schema summary for prompt injection.
|
| 1021 |
-
|
| 1022 |
-
**Step 1: Write the failing tests**
|
| 1023 |
-
|
| 1024 |
-
Append to `tests/test_query_engine.py` (or create if it doesn't exist):
|
| 1025 |
-
|
| 1026 |
-
```python
|
| 1027 |
-
def test_parse_qu_result_with_route_and_sql():
|
| 1028 |
-
from src.query_engine import _parse_qu_result
|
| 1029 |
-
raw = '{"action": "search", "route": "sql", "search_query": "PTS China", "display_query": "PTS scores for China?", "sql_query": "SELECT * FROM t", "reasoning": "data lookup"}'
|
| 1030 |
-
result = _parse_qu_result(raw, "original")
|
| 1031 |
-
assert result["route"] == "sql"
|
| 1032 |
-
assert result["sql_query"] == "SELECT * FROM t"
|
| 1033 |
-
|
| 1034 |
-
|
| 1035 |
-
def test_parse_qu_result_defaults_route_to_vector():
|
| 1036 |
-
from src.query_engine import _parse_qu_result
|
| 1037 |
-
raw = '{"action": "search", "search_query": "test", "display_query": "test?"}'
|
| 1038 |
-
result = _parse_qu_result(raw, "original")
|
| 1039 |
-
assert result["route"] == "vector"
|
| 1040 |
-
assert result.get("sql_query") is None
|
| 1041 |
-
|
| 1042 |
-
|
| 1043 |
-
def test_load_sql_schema_missing_file():
|
| 1044 |
-
from src.query_engine import _load_sql_schema_summary
|
| 1045 |
-
result = _load_sql_schema_summary("/nonexistent/path/sql_db")
|
| 1046 |
-
assert result == ""
|
| 1047 |
-
|
| 1048 |
-
|
| 1049 |
-
def test_load_sql_schema_valid_file(tmp_path):
|
| 1050 |
-
import json
|
| 1051 |
-
from src.query_engine import _load_sql_schema_summary
|
| 1052 |
-
schema_dir = tmp_path / "sql_db"
|
| 1053 |
-
schema_dir.mkdir()
|
| 1054 |
-
schema = {
|
| 1055 |
-
"test_table": {
|
| 1056 |
-
"source_file": "data.csv",
|
| 1057 |
-
"columns": [{"name": "x", "type": "TEXT", "sample": ["a"]}],
|
| 1058 |
-
"row_count": 10,
|
| 1059 |
-
}
|
| 1060 |
-
}
|
| 1061 |
-
(schema_dir / "sql_schemas.json").write_text(json.dumps(schema))
|
| 1062 |
-
result = _load_sql_schema_summary(str(schema_dir))
|
| 1063 |
-
assert "test_table" in result
|
| 1064 |
-
assert "10 rows" in result
|
| 1065 |
-
```
|
| 1066 |
-
|
| 1067 |
-
**Step 2: Run tests to verify they fail**
|
| 1068 |
-
|
| 1069 |
-
Run: `python -m pytest tests/test_query_engine.py::test_parse_qu_result_with_route_and_sql -v`
|
| 1070 |
-
Expected: FAIL
|
| 1071 |
-
|
| 1072 |
-
**Step 3: Edit `src/query_engine.py`**
|
| 1073 |
-
|
| 1074 |
-
**3a.** Add `_load_sql_schema_summary()` function:
|
| 1075 |
-
|
| 1076 |
-
```python
|
| 1077 |
-
import json as _json_mod
|
| 1078 |
-
|
| 1079 |
-
def _load_sql_schema_summary(sql_db_dir: str) -> str:
|
| 1080 |
-
"""Load SQL schema summary from sql_schemas.json if it exists."""
|
| 1081 |
-
import os
|
| 1082 |
-
schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
|
| 1083 |
-
if not os.path.exists(schema_path):
|
| 1084 |
-
return ""
|
| 1085 |
-
try:
|
| 1086 |
-
with open(schema_path, "r", encoding="utf-8") as f:
|
| 1087 |
-
schema = _json_mod.load(f)
|
| 1088 |
-
from src.sql_retriever import build_schema_summary
|
| 1089 |
-
return build_schema_summary(schema)
|
| 1090 |
-
except Exception:
|
| 1091 |
-
return ""
|
| 1092 |
-
```
|
| 1093 |
-
|
| 1094 |
-
**3b.** Update `understand_query()` to load schema and pass to prompt builder:
|
| 1095 |
-
|
| 1096 |
-
```python
|
| 1097 |
-
# After getting domain and max_history, add:
|
| 1098 |
-
sql_enabled = cfg.get("sql", {}).get("enabled", True)
|
| 1099 |
-
schema_summary = ""
|
| 1100 |
-
if sql_enabled:
|
| 1101 |
-
sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
|
| 1102 |
-
if not os.path.isabs(sql_db_dir):
|
| 1103 |
-
from pathlib import Path as _Path
|
| 1104 |
-
project_root = _Path(__file__).resolve().parent.parent
|
| 1105 |
-
sql_db_dir = os.path.join(str(project_root), sql_db_dir)
|
| 1106 |
-
schema_summary = _load_sql_schema_summary(sql_db_dir)
|
| 1107 |
-
|
| 1108 |
-
prompt = build_query_understanding_prompt(
|
| 1109 |
-
user_query, domain, history, sql_schema_summary=schema_summary,
|
| 1110 |
-
)
|
| 1111 |
-
```
|
| 1112 |
-
|
| 1113 |
-
Add `import os` at the top if not already there.
|
| 1114 |
-
|
| 1115 |
-
**3c.** Update `_parse_qu_result()` to extract `route` and `sql_query`:
|
| 1116 |
-
|
| 1117 |
-
In the `action == "search"` branch, after extracting `search_query` and `display_query`, add:
|
| 1118 |
-
|
| 1119 |
-
```python
|
| 1120 |
-
route = parsed.get("route", "vector")
|
| 1121 |
-
if route not in ("sql", "vector", "both"):
|
| 1122 |
-
route = "vector"
|
| 1123 |
-
sql_query = parsed.get("sql_query") if route in ("sql", "both") else None
|
| 1124 |
-
```
|
| 1125 |
-
|
| 1126 |
-
And include both in the return dict:
|
| 1127 |
-
|
| 1128 |
-
```python
|
| 1129 |
-
return {
|
| 1130 |
-
"action": "search",
|
| 1131 |
-
"search_query": search_query,
|
| 1132 |
-
"display_query": display_query,
|
| 1133 |
-
"original_query": original_query,
|
| 1134 |
-
"route": route,
|
| 1135 |
-
"sql_query": sql_query,
|
| 1136 |
-
}
|
| 1137 |
-
```
|
| 1138 |
-
|
| 1139 |
-
Also update all other return paths (fallbacks, clarify) to include `"route": "vector"` and `"sql_query": None`.
|
| 1140 |
-
|
| 1141 |
-
**Step 4: Run tests to verify they pass**
|
| 1142 |
-
|
| 1143 |
-
Run: `python -m pytest tests/test_query_engine.py -v`
|
| 1144 |
-
Expected: All PASS
|
| 1145 |
-
|
| 1146 |
-
Run: `python -m pytest tests/ -v`
|
| 1147 |
-
Expected: All PASS
|
| 1148 |
-
|
| 1149 |
-
**Step 5: Commit**
|
| 1150 |
-
|
| 1151 |
-
```bash
|
| 1152 |
-
git add src/query_engine.py tests/test_query_engine.py
|
| 1153 |
-
git commit -m "feat: add SQL schema loading and route/sql_query parsing to query engine"
|
| 1154 |
-
```
|
| 1155 |
-
|
| 1156 |
-
---
|
| 1157 |
-
|
| 1158 |
-
### Task 7: Retriever integration — Routing, fallback, SQL context merging
|
| 1159 |
-
|
| 1160 |
-
**Files:**
|
| 1161 |
-
- Modify: `src/retriever.py:202-227` (the `retrieve()` function)
|
| 1162 |
-
- Modify: `src/retriever.py:171-199` (`build_combined_context()`)
|
| 1163 |
-
- Modify: `tests/test_retriever.py`
|
| 1164 |
-
|
| 1165 |
-
**Context:** Read `src/retriever.py` before editing. The `retrieve()` function currently takes `(query, cfg)` and returns `{context, db_results, web_results, has_sources}`. We need to add `route` and `sql_query` parameters, implement routing with fallback, and update `build_combined_context()` to accept SQL results.
|
| 1166 |
-
|
| 1167 |
-
**Step 1: Write the failing tests**
|
| 1168 |
-
|
| 1169 |
-
Append to `tests/test_retriever.py`:
|
| 1170 |
-
|
| 1171 |
-
```python
|
| 1172 |
-
def test_build_combined_context_with_sql():
|
| 1173 |
-
from src.retriever import build_combined_context
|
| 1174 |
-
sql = "=== SQL Query Results (PRIMARY) ===\n[CHUNK-SQL-001] x = 1"
|
| 1175 |
-
db = [{"text": "data", "metadata": {"source": "f.pdf", "dataset": "d", "page": "1"}, "distance": 0.1}]
|
| 1176 |
-
result = build_combined_context(db, [], sql_context=sql)
|
| 1177 |
-
assert "SQL Query Results" in result
|
| 1178 |
-
assert "CHUNK-LOCAL" in result
|
| 1179 |
-
|
| 1180 |
-
|
| 1181 |
-
def test_build_combined_context_sql_only():
|
| 1182 |
-
from src.retriever import build_combined_context
|
| 1183 |
-
sql = "=== SQL Query Results (PRIMARY) ===\n[CHUNK-SQL-001] x = 1"
|
| 1184 |
-
result = build_combined_context([], [], sql_context=sql)
|
| 1185 |
-
assert "SQL Query Results" in result
|
| 1186 |
-
|
| 1187 |
-
|
| 1188 |
-
def test_build_combined_context_sql_no_sources():
|
| 1189 |
-
from src.retriever import build_combined_context
|
| 1190 |
-
result = build_combined_context([], [], sql_context="")
|
| 1191 |
-
assert "No relevant" in result or "don't have" in result
|
| 1192 |
-
```
|
| 1193 |
-
|
| 1194 |
-
**Step 2: Run tests to verify they fail**
|
| 1195 |
-
|
| 1196 |
-
Run: `python -m pytest tests/test_retriever.py::test_build_combined_context_with_sql -v`
|
| 1197 |
-
Expected: FAIL — `build_combined_context() got an unexpected keyword argument 'sql_context'`
|
| 1198 |
-
|
| 1199 |
-
**Step 3: Edit `src/retriever.py`**
|
| 1200 |
-
|
| 1201 |
-
**3a.** Update `build_combined_context()` to accept `sql_context`:
|
| 1202 |
-
|
| 1203 |
-
```python
|
| 1204 |
-
def build_combined_context(
|
| 1205 |
-
db_results: list[dict], web_results: list[dict], sql_context: str = "",
|
| 1206 |
-
) -> str:
|
| 1207 |
-
"""Build combined context string from local, SQL, and web results."""
|
| 1208 |
-
db_context = format_db_results_as_context(db_results)
|
| 1209 |
-
web_context = format_web_results_as_context(web_results)
|
| 1210 |
-
|
| 1211 |
-
has_local = bool(db_results) or bool(sql_context)
|
| 1212 |
-
has_web = bool(web_results)
|
| 1213 |
-
|
| 1214 |
-
if not has_local and not has_web:
|
| 1215 |
-
return NO_SOURCES_REFUSAL
|
| 1216 |
-
|
| 1217 |
-
parts = []
|
| 1218 |
-
|
| 1219 |
-
if has_local and not has_web:
|
| 1220 |
-
parts.append("ONLY use the local sources below. Do not add any outside knowledge.\n")
|
| 1221 |
-
|
| 1222 |
-
if has_local and has_web:
|
| 1223 |
-
parts.append(
|
| 1224 |
-
"IMPORTANT: Local document sources are the PRIMARY authority. "
|
| 1225 |
-
"Web sources are SUPPLEMENTARY only. If any web source contradicts "
|
| 1226 |
-
"a local document, trust the local document.\n"
|
| 1227 |
-
)
|
| 1228 |
-
|
| 1229 |
-
if not has_local and has_web:
|
| 1230 |
-
parts.append(
|
| 1231 |
-
"NOTE: No local documents are indexed. All sources below come from "
|
| 1232 |
-
"academic web search. You MUST include the URL link for every source cited. "
|
| 1233 |
-
"State clearly that you have no curated local sources.\n"
|
| 1234 |
-
)
|
| 1235 |
-
|
| 1236 |
-
if sql_context:
|
| 1237 |
-
parts.append(sql_context)
|
| 1238 |
-
if db_context:
|
| 1239 |
-
parts.append(db_context)
|
| 1240 |
-
if web_context:
|
| 1241 |
-
parts.append(web_context)
|
| 1242 |
-
|
| 1243 |
-
return "\n\n".join(parts)
|
| 1244 |
-
```
|
| 1245 |
-
|
| 1246 |
-
**3b.** Update `retrieve()` to accept `route` and `sql_query`, implement routing:
|
| 1247 |
-
|
| 1248 |
-
```python
|
| 1249 |
-
def retrieve(
|
| 1250 |
-
query: str, cfg: dict, route: str = "vector", sql_query: str = None,
|
| 1251 |
-
) -> dict:
|
| 1252 |
-
"""Run retrieval with SQL/vector routing and fallback.
|
| 1253 |
-
|
| 1254 |
-
Returns dict with: context, db_results, web_results, sql_results, has_sources.
|
| 1255 |
-
"""
|
| 1256 |
-
sql_enabled = cfg.get("sql", {}).get("enabled", True)
|
| 1257 |
-
sql_context = ""
|
| 1258 |
-
sql_rows = []
|
| 1259 |
-
|
| 1260 |
-
# ── SQL retrieval ────────────────────────────────────────────────
|
| 1261 |
-
if sql_enabled and route in ("sql", "both") and sql_query:
|
| 1262 |
-
from src.sql_retriever import execute_sql_query, format_sql_results_as_context, _lookup_source_file
|
| 1263 |
-
sql_rows = execute_sql_query(sql_query, cfg)
|
| 1264 |
-
if sql_rows:
|
| 1265 |
-
# Look up source file from schema
|
| 1266 |
-
import json as _json
|
| 1267 |
-
import os
|
| 1268 |
-
sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
|
| 1269 |
-
if not os.path.isabs(sql_db_dir):
|
| 1270 |
-
from pathlib import Path
|
| 1271 |
-
project_root = Path(__file__).resolve().parent.parent
|
| 1272 |
-
sql_db_dir = os.path.join(str(project_root), sql_db_dir)
|
| 1273 |
-
schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
|
| 1274 |
-
schema = {}
|
| 1275 |
-
if os.path.exists(schema_path):
|
| 1276 |
-
try:
|
| 1277 |
-
with open(schema_path) as f:
|
| 1278 |
-
schema = _json.load(f)
|
| 1279 |
-
except Exception:
|
| 1280 |
-
pass
|
| 1281 |
-
# Extract table name from query for source lookup
|
| 1282 |
-
table_name = _extract_table_from_query(sql_query)
|
| 1283 |
-
source_file = _lookup_source_file(table_name, schema)
|
| 1284 |
-
sql_context = format_sql_results_as_context(sql_rows, sql_query, source_file)
|
| 1285 |
-
|
| 1286 |
-
# ── Vector retrieval ─────────────────────────────────────────────
|
| 1287 |
-
db_results = []
|
| 1288 |
-
if route in ("vector", "both") or (route == "sql" and not sql_rows):
|
| 1289 |
-
db_results = retrieve_from_vectordb(query, cfg)
|
| 1290 |
-
|
| 1291 |
-
# ── Fallback: vector found nothing, try SQL ──────────────────────
|
| 1292 |
-
if sql_enabled and not db_results and not sql_rows and route == "vector" and sql_query:
|
| 1293 |
-
from src.sql_retriever import execute_sql_query, format_sql_results_as_context, _lookup_source_file
|
| 1294 |
-
sql_rows = execute_sql_query(sql_query, cfg)
|
| 1295 |
-
if sql_rows:
|
| 1296 |
-
import json as _json, os
|
| 1297 |
-
sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
|
| 1298 |
-
if not os.path.isabs(sql_db_dir):
|
| 1299 |
-
from pathlib import Path
|
| 1300 |
-
project_root = Path(__file__).resolve().parent.parent
|
| 1301 |
-
sql_db_dir = os.path.join(str(project_root), sql_db_dir)
|
| 1302 |
-
schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
|
| 1303 |
-
schema = {}
|
| 1304 |
-
if os.path.exists(schema_path):
|
| 1305 |
-
try:
|
| 1306 |
-
with open(schema_path) as f:
|
| 1307 |
-
schema = _json.load(f)
|
| 1308 |
-
except Exception:
|
| 1309 |
-
pass
|
| 1310 |
-
table_name = _extract_table_from_query(sql_query)
|
| 1311 |
-
source_file = _lookup_source_file(table_name, schema)
|
| 1312 |
-
sql_context = format_sql_results_as_context(sql_rows, sql_query, source_file)
|
| 1313 |
-
|
| 1314 |
-
# ── Web search ───────────────────────────────────────────────────
|
| 1315 |
-
web_enabled = cfg.get("web_search", {}).get("enabled", False)
|
| 1316 |
-
web_results = []
|
| 1317 |
-
if web_enabled:
|
| 1318 |
-
backend = cfg.get("web_search", {}).get("backend", "semantic_scholar")
|
| 1319 |
-
max_results = cfg.get("web_search", {}).get("max_results", 5)
|
| 1320 |
-
if not db_results and not sql_rows:
|
| 1321 |
-
max_results *= 2
|
| 1322 |
-
web_results = search(query, backend=backend, limit=max_results)
|
| 1323 |
-
|
| 1324 |
-
combined_context = build_combined_context(db_results, web_results, sql_context=sql_context)
|
| 1325 |
-
has_sources = bool(db_results) or bool(web_results) or bool(sql_rows)
|
| 1326 |
-
|
| 1327 |
-
return {
|
| 1328 |
-
"context": combined_context,
|
| 1329 |
-
"db_results": db_results,
|
| 1330 |
-
"web_results": web_results,
|
| 1331 |
-
"sql_results": sql_rows,
|
| 1332 |
-
"has_sources": has_sources,
|
| 1333 |
-
}
|
| 1334 |
-
```
|
| 1335 |
-
|
| 1336 |
-
**3c.** Add helper to extract table name from SQL query:
|
| 1337 |
-
|
| 1338 |
-
```python
|
| 1339 |
-
def _extract_table_from_query(sql_query: str) -> str:
|
| 1340 |
-
"""Extract the first table name from a SQL query (best-effort)."""
|
| 1341 |
-
match = re.search(r'\bFROM\s+"?(\w+)"?', sql_query, re.IGNORECASE)
|
| 1342 |
-
return match.group(1) if match else ""
|
| 1343 |
-
```
|
| 1344 |
-
|
| 1345 |
-
**Step 4: Run tests to verify they pass**
|
| 1346 |
-
|
| 1347 |
-
Run: `python -m pytest tests/test_retriever.py -v`
|
| 1348 |
-
Expected: All PASS
|
| 1349 |
-
|
| 1350 |
-
Run: `python -m pytest tests/ -v`
|
| 1351 |
-
Expected: All PASS
|
| 1352 |
-
|
| 1353 |
-
**Step 5: Commit**
|
| 1354 |
-
|
| 1355 |
-
```bash
|
| 1356 |
-
git add src/retriever.py tests/test_retriever.py
|
| 1357 |
-
git commit -m "feat: add SQL routing, fallback, and context merging to retriever"
|
| 1358 |
-
```
|
| 1359 |
-
|
| 1360 |
-
---
|
| 1361 |
-
|
| 1362 |
-
### Task 8: Ingestion integration — Call SQL ingest during pipeline
|
| 1363 |
-
|
| 1364 |
-
**Files:**
|
| 1365 |
-
- Modify: `src/ingest.py:145-219` (the `ingest_documents()` function)
|
| 1366 |
-
|
| 1367 |
-
**Context:** Read `src/ingest.py`. After the existing ChromaDB ingestion loop, add a call to `ingest_to_sql()` for the same file list.
|
| 1368 |
-
|
| 1369 |
-
**Step 1: Write the failing test**
|
| 1370 |
-
|
| 1371 |
-
Append to `tests/test_sql_ingest.py`:
|
| 1372 |
-
|
| 1373 |
-
```python
|
| 1374 |
-
def test_ingest_documents_calls_sql_ingest(tmp_path):
|
| 1375 |
-
"""Integration test: ingest_documents should also populate SQLite."""
|
| 1376 |
-
import os
|
| 1377 |
-
from pathlib import Path
|
| 1378 |
-
|
| 1379 |
-
# Set up knowledge base with a CSV
|
| 1380 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 1381 |
-
ds_dir = kb_dir / "testds"
|
| 1382 |
-
ds_dir.mkdir(parents=True)
|
| 1383 |
-
csv_file = ds_dir / "scores.csv"
|
| 1384 |
-
csv_file.write_text("Country,Year,Score\nChina,2005,4.0\nIndia,2005,3.0\n")
|
| 1385 |
-
|
| 1386 |
-
# Minimal config
|
| 1387 |
-
cfg = {
|
| 1388 |
-
"paths": {
|
| 1389 |
-
"knowledge_base": str(kb_dir),
|
| 1390 |
-
"vector_db": str(tmp_path / "chroma_db"),
|
| 1391 |
-
"sql_db": str(tmp_path / "sql_db"),
|
| 1392 |
-
},
|
| 1393 |
-
"retrieval": {"chunk_size": 1000, "chunk_overlap": 100},
|
| 1394 |
-
"embeddings": {"provider": "local"},
|
| 1395 |
-
"sql": {"enabled": True},
|
| 1396 |
-
}
|
| 1397 |
-
|
| 1398 |
-
from src.ingest import ingest_documents
|
| 1399 |
-
count = ingest_documents(cfg, documents_dir=str(kb_dir))
|
| 1400 |
-
|
| 1401 |
-
# Verify ChromaDB got chunks
|
| 1402 |
-
assert count > 0
|
| 1403 |
-
|
| 1404 |
-
# Verify SQLite DB was created
|
| 1405 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 1406 |
-
assert db_path.exists()
|
| 1407 |
-
|
| 1408 |
-
# Verify schema registry
|
| 1409 |
-
schema_path = tmp_path / "sql_db" / "sql_schemas.json"
|
| 1410 |
-
assert schema_path.exists()
|
| 1411 |
-
```
|
| 1412 |
-
|
| 1413 |
-
**Step 2: Run test to verify it fails**
|
| 1414 |
-
|
| 1415 |
-
Run: `python -m pytest tests/test_sql_ingest.py::test_ingest_documents_calls_sql_ingest -v`
|
| 1416 |
-
Expected: FAIL — SQLite DB not created (sql_ingest not called yet)
|
| 1417 |
-
|
| 1418 |
-
**Step 3: Edit `src/ingest.py`**
|
| 1419 |
-
|
| 1420 |
-
At the end of `ingest_documents()`, after the ChromaDB ingestion loop and before the final print, add:
|
| 1421 |
-
|
| 1422 |
-
```python
|
| 1423 |
-
# ── SQL ingestion for tabular files ──────────────────────────────
|
| 1424 |
-
sql_enabled = cfg.get("sql", {}).get("enabled", True)
|
| 1425 |
-
if sql_enabled:
|
| 1426 |
-
from src.sql_ingest import ingest_to_sql, SQL_EXTENSIONS
|
| 1427 |
-
tabular_count = sum(1 for f, _ in files if f.suffix.lower() in SQL_EXTENSIONS)
|
| 1428 |
-
if tabular_count > 0:
|
| 1429 |
-
print(f"\nIngesting {tabular_count} tabular file(s) into SQLite...")
|
| 1430 |
-
try:
|
| 1431 |
-
schema = ingest_to_sql(files, documents_dir, cfg)
|
| 1432 |
-
print(f"SQL ingestion complete: {len(schema)} table(s).")
|
| 1433 |
-
except Exception as e:
|
| 1434 |
-
print(f"SQL ingestion error (non-fatal): {e}")
|
| 1435 |
-
```
|
| 1436 |
-
|
| 1437 |
-
**Step 4: Run tests to verify they pass**
|
| 1438 |
-
|
| 1439 |
-
Run: `python -m pytest tests/test_sql_ingest.py -v`
|
| 1440 |
-
Expected: All PASS
|
| 1441 |
-
|
| 1442 |
-
Run: `python -m pytest tests/ -v`
|
| 1443 |
-
Expected: All PASS
|
| 1444 |
-
|
| 1445 |
-
**Step 5: Commit**
|
| 1446 |
-
|
| 1447 |
-
```bash
|
| 1448 |
-
git add src/ingest.py tests/test_sql_ingest.py
|
| 1449 |
-
git commit -m "feat: integrate SQL ingestion into the main ingest pipeline"
|
| 1450 |
-
```
|
| 1451 |
-
|
| 1452 |
-
---
|
| 1453 |
-
|
| 1454 |
-
### Task 9: App integration — Pass route/sql_query through CLI and Web UI
|
| 1455 |
-
|
| 1456 |
-
**Files:**
|
| 1457 |
-
- Modify: `app_cli.py:315-325` (retrieve call)
|
| 1458 |
-
- Modify: `app_web.py:194-195` (retrieve call)
|
| 1459 |
-
|
| 1460 |
-
**Context:** Read both files. Both call `retrieve(search_query, cfg)`. They need to also pass `route` and `sql_query` from the QU result.
|
| 1461 |
-
|
| 1462 |
-
**Step 1: No new tests needed** — this is a wiring change. The existing test suite plus manual testing covers it.
|
| 1463 |
-
|
| 1464 |
-
**Step 2: Edit `app_cli.py`**
|
| 1465 |
-
|
| 1466 |
-
In the RAG pipeline section, after the QU result is finalized (~line 315), extract route and sql_query:
|
| 1467 |
-
|
| 1468 |
-
```python
|
| 1469 |
-
search_query = qu_result.get("search_query", user_input)
|
| 1470 |
-
display_query = qu_result.get("display_query", user_input)
|
| 1471 |
-
route = qu_result.get("route", "vector")
|
| 1472 |
-
sql_query = qu_result.get("sql_query")
|
| 1473 |
-
```
|
| 1474 |
-
|
| 1475 |
-
Then update the retrieve call (~line 325):
|
| 1476 |
-
|
| 1477 |
-
```python
|
| 1478 |
-
retrieval_result = retrieve(search_query, effective_cfg, route=route, sql_query=sql_query)
|
| 1479 |
-
```
|
| 1480 |
-
|
| 1481 |
-
Also show SQL route indicator to user (after the "Searching for:" line):
|
| 1482 |
-
|
| 1483 |
-
```python
|
| 1484 |
-
if route in ("sql", "both"):
|
| 1485 |
-
console.print(f"[dim]Using SQL query for structured data[/dim]")
|
| 1486 |
-
```
|
| 1487 |
-
|
| 1488 |
-
**Step 3: Edit `app_web.py`**
|
| 1489 |
-
|
| 1490 |
-
Similarly, after QU result is finalized (~line 183):
|
| 1491 |
-
|
| 1492 |
-
```python
|
| 1493 |
-
search_query = qu_result.get("search_query", combined)
|
| 1494 |
-
display_query = qu_result.get("display_query", original_query)
|
| 1495 |
-
route = qu_result.get("route", "vector")
|
| 1496 |
-
sql_query = qu_result.get("sql_query")
|
| 1497 |
-
```
|
| 1498 |
-
|
| 1499 |
-
Update the retrieve call (~line 195):
|
| 1500 |
-
|
| 1501 |
-
```python
|
| 1502 |
-
retrieval_result = retrieve(search_query, cfg, route=route, sql_query=sql_query)
|
| 1503 |
-
```
|
| 1504 |
-
|
| 1505 |
-
Update the source summary to include SQL results:
|
| 1506 |
-
|
| 1507 |
-
```python
|
| 1508 |
-
n_sql = len(retrieval_result.get("sql_results", []))
|
| 1509 |
-
```
|
| 1510 |
-
|
| 1511 |
-
And display it.
|
| 1512 |
-
|
| 1513 |
-
**Step 4: Run full test suite**
|
| 1514 |
-
|
| 1515 |
-
Run: `python -m pytest tests/ -v`
|
| 1516 |
-
Expected: All PASS
|
| 1517 |
-
|
| 1518 |
-
**Step 5: Commit**
|
| 1519 |
-
|
| 1520 |
-
```bash
|
| 1521 |
-
git add app_cli.py app_web.py
|
| 1522 |
-
git commit -m "feat: wire SQL routing through CLI and web UI"
|
| 1523 |
-
```
|
| 1524 |
-
|
| 1525 |
-
---
|
| 1526 |
-
|
| 1527 |
-
### Task 10: Config, gitignore, and documentation
|
| 1528 |
-
|
| 1529 |
-
**Files:**
|
| 1530 |
-
- Modify: `setup.py` (`generate_config()`)
|
| 1531 |
-
- Modify: `.gitignore`
|
| 1532 |
-
- Modify: `CLAUDE.md`
|
| 1533 |
-
- Modify: `README.md`
|
| 1534 |
-
|
| 1535 |
-
**Step 1: Edit `setup.py`**
|
| 1536 |
-
|
| 1537 |
-
In `generate_config()`, add to the config dict:
|
| 1538 |
-
|
| 1539 |
-
```python
|
| 1540 |
-
"paths": {
|
| 1541 |
-
...
|
| 1542 |
-
"sql_db": "sql_db",
|
| 1543 |
-
},
|
| 1544 |
-
"sql": {
|
| 1545 |
-
"enabled": True,
|
| 1546 |
-
"max_rows": 200,
|
| 1547 |
-
},
|
| 1548 |
-
```
|
| 1549 |
-
|
| 1550 |
-
**Step 2: Edit `.gitignore`**
|
| 1551 |
-
|
| 1552 |
-
Add `sql_db/` line after `chroma_db/`.
|
| 1553 |
-
|
| 1554 |
-
**Step 3: Edit `CLAUDE.md`**
|
| 1555 |
-
|
| 1556 |
-
- Add `paths.sql_db`, `sql.enabled`, `sql.max_rows` to the config table
|
| 1557 |
-
- Add SQL layer to architecture description
|
| 1558 |
-
- Update project structure to include `src/sql_ingest.py`, `src/sql_retriever.py`, `sql_db/`
|
| 1559 |
-
- Update test count
|
| 1560 |
-
- Add `retrieval.hybrid_search` to config table if missing
|
| 1561 |
-
|
| 1562 |
-
**Step 4: Edit `README.md`**
|
| 1563 |
-
|
| 1564 |
-
- Add SQL layer feature description
|
| 1565 |
-
- Add `sql` section to config reference
|
| 1566 |
-
- Add `sql_db/` to gitignored directories
|
| 1567 |
-
|
| 1568 |
-
**Step 5: Update test for setup.py**
|
| 1569 |
-
|
| 1570 |
-
In `tests/test_setup.py`, add assertions:
|
| 1571 |
-
|
| 1572 |
-
```python
|
| 1573 |
-
assert cfg["paths"]["sql_db"] == "sql_db"
|
| 1574 |
-
assert cfg["sql"]["enabled"] is True
|
| 1575 |
-
assert cfg["sql"]["max_rows"] == 200
|
| 1576 |
-
```
|
| 1577 |
-
|
| 1578 |
-
**Step 6: Run tests**
|
| 1579 |
-
|
| 1580 |
-
Run: `python -m pytest tests/ -v`
|
| 1581 |
-
Expected: All PASS
|
| 1582 |
-
|
| 1583 |
-
**Step 7: Commit**
|
| 1584 |
-
|
| 1585 |
-
```bash
|
| 1586 |
-
git add setup.py .gitignore CLAUDE.md README.md tests/test_setup.py
|
| 1587 |
-
git commit -m "feat: add SQL layer config, gitignore, and documentation"
|
| 1588 |
-
```
|
| 1589 |
-
|
| 1590 |
-
---
|
| 1591 |
-
|
| 1592 |
-
### Task 11: Final integration test and cleanup
|
| 1593 |
-
|
| 1594 |
-
**Files:**
|
| 1595 |
-
- Create: `tests/test_sql_integration.py`
|
| 1596 |
-
|
| 1597 |
-
**Step 1: Write an end-to-end integration test**
|
| 1598 |
-
|
| 1599 |
-
```python
|
| 1600 |
-
# tests/test_sql_integration.py
|
| 1601 |
-
"""End-to-end integration test for the SQL layer."""
|
| 1602 |
-
import json
|
| 1603 |
-
import sqlite3
|
| 1604 |
-
import pytest
|
| 1605 |
-
|
| 1606 |
-
|
| 1607 |
-
def test_sql_roundtrip(tmp_path):
|
| 1608 |
-
"""Test: ingest CSV -> SQL -> retrieve -> format context."""
|
| 1609 |
-
from pathlib import Path
|
| 1610 |
-
from src.sql_ingest import ingest_to_sql
|
| 1611 |
-
from src.sql_retriever import execute_sql_query, format_sql_results_as_context, build_schema_summary
|
| 1612 |
-
|
| 1613 |
-
# Create test data
|
| 1614 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 1615 |
-
ds_dir = kb_dir / "pts"
|
| 1616 |
-
ds_dir.mkdir(parents=True)
|
| 1617 |
-
csv_file = ds_dir / "scores.csv"
|
| 1618 |
-
csv_file.write_text(
|
| 1619 |
-
"Country,Year,PTS_A\n"
|
| 1620 |
-
"China,2000,4.0\n"
|
| 1621 |
-
"China,2001,4.0\n"
|
| 1622 |
-
"China,2002,3.5\n"
|
| 1623 |
-
"India,2000,3.0\n"
|
| 1624 |
-
"India,2001,3.0\n"
|
| 1625 |
-
)
|
| 1626 |
-
|
| 1627 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 1628 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}, "sql": {"max_rows": 200}}
|
| 1629 |
-
files = [(Path(csv_file), "pts")]
|
| 1630 |
-
|
| 1631 |
-
# Ingest
|
| 1632 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 1633 |
-
assert len(schema) == 1
|
| 1634 |
-
table_name = list(schema.keys())[0]
|
| 1635 |
-
|
| 1636 |
-
# Build schema summary
|
| 1637 |
-
summary = build_schema_summary(schema)
|
| 1638 |
-
assert "Country" in summary
|
| 1639 |
-
assert "PTS_A" in summary
|
| 1640 |
-
|
| 1641 |
-
# Execute a query
|
| 1642 |
-
sql = f'SELECT Country, Year, PTS_A FROM "{table_name}" WHERE Country = \'China\' ORDER BY Year'
|
| 1643 |
-
rows = execute_sql_query(sql, cfg)
|
| 1644 |
-
assert len(rows) == 3
|
| 1645 |
-
assert all(r["Country"] == "China" for r in rows)
|
| 1646 |
-
|
| 1647 |
-
# Format as context
|
| 1648 |
-
context = format_sql_results_as_context(rows, sql, schema[table_name]["source_file"])
|
| 1649 |
-
assert "[CHUNK-SQL-001]" in context
|
| 1650 |
-
assert "China" in context
|
| 1651 |
-
assert "pts/scores.csv" in context
|
| 1652 |
-
|
| 1653 |
-
|
| 1654 |
-
def test_sql_aggregation(tmp_path):
|
| 1655 |
-
"""Test: aggregation queries work."""
|
| 1656 |
-
from pathlib import Path
|
| 1657 |
-
from src.sql_ingest import ingest_to_sql
|
| 1658 |
-
from src.sql_retriever import execute_sql_query
|
| 1659 |
-
|
| 1660 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 1661 |
-
ds_dir = kb_dir / "ds"
|
| 1662 |
-
ds_dir.mkdir(parents=True)
|
| 1663 |
-
csv_file = ds_dir / "data.csv"
|
| 1664 |
-
csv_file.write_text("Country,Year,Score\nA,2000,1\nB,2000,2\nC,2000,3\n")
|
| 1665 |
-
|
| 1666 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 1667 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}, "sql": {"max_rows": 200}}
|
| 1668 |
-
files = [(Path(csv_file), "ds")]
|
| 1669 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 1670 |
-
table_name = list(schema.keys())[0]
|
| 1671 |
-
|
| 1672 |
-
# Count
|
| 1673 |
-
rows = execute_sql_query(f'SELECT COUNT(*) as cnt FROM "{table_name}"', cfg)
|
| 1674 |
-
assert rows[0]["cnt"] == 3
|
| 1675 |
-
|
| 1676 |
-
# Average
|
| 1677 |
-
rows = execute_sql_query(f'SELECT AVG(Score) as avg_score FROM "{table_name}"', cfg)
|
| 1678 |
-
assert rows[0]["avg_score"] == 2.0
|
| 1679 |
-
```
|
| 1680 |
-
|
| 1681 |
-
**Step 2: Run integration tests**
|
| 1682 |
-
|
| 1683 |
-
Run: `python -m pytest tests/test_sql_integration.py -v`
|
| 1684 |
-
Expected: 2 PASSED
|
| 1685 |
-
|
| 1686 |
-
**Step 3: Run the full test suite**
|
| 1687 |
-
|
| 1688 |
-
Run: `python -m pytest tests/ -v`
|
| 1689 |
-
Expected: All PASS. Note the new test count.
|
| 1690 |
-
|
| 1691 |
-
**Step 4: Commit**
|
| 1692 |
-
|
| 1693 |
-
```bash
|
| 1694 |
-
git add tests/test_sql_integration.py
|
| 1695 |
-
git commit -m "test: add end-to-end SQL layer integration tests"
|
| 1696 |
-
```
|
| 1697 |
-
|
| 1698 |
-
**Step 5: Update test count in CLAUDE.md** if needed.
|
| 1699 |
-
|
| 1700 |
-
---
|
| 1701 |
-
|
| 1702 |
-
## Summary
|
| 1703 |
-
|
| 1704 |
-
| Task | What it does | New/Modified files |
|
| 1705 |
-
|---|---|---|
|
| 1706 |
-
| 1 | SQL ingest helpers (naming, types, samples) | `src/sql_ingest.py`, `tests/test_sql_ingest.py` |
|
| 1707 |
-
| 2 | Core SQL ingestion function | `src/sql_ingest.py`, `tests/test_sql_ingest.py` |
|
| 1708 |
-
| 3 | SQL retriever (validation + execution) | `src/sql_retriever.py`, `tests/test_sql_retriever.py` |
|
| 1709 |
-
| 4 | SQL result formatting + schema summary | `src/sql_retriever.py`, `tests/test_sql_retriever.py` |
|
| 1710 |
-
| 5 | Prompt updates (QU, system, verification) | `src/prompts.py`, `tests/test_prompts.py` |
|
| 1711 |
-
| 6 | Query engine (schema loading, route parsing) | `src/query_engine.py`, `tests/test_query_engine.py` |
|
| 1712 |
-
| 7 | Retriever (routing, fallback, context merge) | `src/retriever.py`, `tests/test_retriever.py` |
|
| 1713 |
-
| 8 | Ingestion integration | `src/ingest.py`, `tests/test_sql_ingest.py` |
|
| 1714 |
-
| 9 | App integration (CLI + Web UI) | `app_cli.py`, `app_web.py` |
|
| 1715 |
-
| 10 | Config, gitignore, docs | `setup.py`, `.gitignore`, `CLAUDE.md`, `README.md`, `tests/test_setup.py` |
|
| 1716 |
-
| 11 | Integration tests + cleanup | `tests/test_sql_integration.py` |
|
| 1717 |
-
|
| 1718 |
-
**Estimated new tests:** ~30 tests across 4 new test files
|
| 1719 |
-
**Total test count after:** ~97 tests
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/ingest.py
CHANGED
|
@@ -201,14 +201,15 @@ def ingest_documents(cfg: dict = None, documents_dir: str = None) -> int:
|
|
| 201 |
print(f" No text extracted, skipping.")
|
| 202 |
continue
|
| 203 |
|
| 204 |
-
source_name =
|
| 205 |
chunks = chunk_documents(pages, source_name, dataset_name, chunk_size, chunk_overlap)
|
| 206 |
print(f" -> {len(chunks)} chunks")
|
| 207 |
|
| 208 |
batch_size = 100
|
| 209 |
for i in range(0, len(chunks), batch_size):
|
| 210 |
batch = chunks[i:i + batch_size]
|
| 211 |
-
|
|
|
|
| 212 |
documents = [c.get("text", "") for c in batch]
|
| 213 |
metadatas = [c.get("metadata", {}) for c in batch]
|
| 214 |
collection.add(ids=ids, documents=documents, metadatas=metadatas)
|
|
|
|
| 201 |
print(f" No text extracted, skipping.")
|
| 202 |
continue
|
| 203 |
|
| 204 |
+
source_name = file_path.relative_to(Path(documents_dir)).as_posix()
|
| 205 |
chunks = chunk_documents(pages, source_name, dataset_name, chunk_size, chunk_overlap)
|
| 206 |
print(f" -> {len(chunks)} chunks")
|
| 207 |
|
| 208 |
batch_size = 100
|
| 209 |
for i in range(0, len(chunks), batch_size):
|
| 210 |
batch = chunks[i:i + batch_size]
|
| 211 |
+
safe_id_prefix = source_name.replace("/", "_").replace(" ", "_")
|
| 212 |
+
ids = [f"{safe_id_prefix}_{i + j}" for j in range(len(batch))]
|
| 213 |
documents = [c.get("text", "") for c in batch]
|
| 214 |
metadatas = [c.get("metadata", {}) for c in batch]
|
| 215 |
collection.add(ids=ids, documents=documents, metadatas=metadatas)
|
src/kb_meta.py
CHANGED
|
@@ -17,15 +17,12 @@ import json
|
|
| 17 |
import os
|
| 18 |
from pathlib import Path
|
| 19 |
|
|
|
|
|
|
|
| 20 |
META_CHUNK_ID = "kb_meta_overview_001"
|
| 21 |
META_SOURCE = "Knowledge Base Overview"
|
| 22 |
META_DATASET = "meta"
|
| 23 |
|
| 24 |
-
|
| 25 |
-
def _escape_braces(text: str) -> str:
|
| 26 |
-
"""Escape curly braces in user-supplied text for safe use with str.format()."""
|
| 27 |
-
return text.replace("{", "{{").replace("}", "}}")
|
| 28 |
-
|
| 29 |
_META_GENERATION_PROMPT = """\
|
| 30 |
You are a knowledge base analyst. Given the following information about
|
| 31 |
documents and data tables in a research knowledge base, write a concise
|
|
|
|
| 17 |
import os
|
| 18 |
from pathlib import Path
|
| 19 |
|
| 20 |
+
from src.prompts import _escape_braces
|
| 21 |
+
|
| 22 |
META_CHUNK_ID = "kb_meta_overview_001"
|
| 23 |
META_SOURCE = "Knowledge Base Overview"
|
| 24 |
META_DATASET = "meta"
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
_META_GENERATION_PROMPT = """\
|
| 27 |
You are a knowledge base analyst. Given the following information about
|
| 28 |
documents and data tables in a research knowledge base, write a concise
|
tests/__init__.py
DELETED
|
File without changes
|
tests/test_app_cli.py
DELETED
|
@@ -1,16 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_cli_commands_exist():
|
| 4 |
-
from app_cli import handle_command
|
| 5 |
-
assert callable(handle_command)
|
| 6 |
-
|
| 7 |
-
def test_handle_command_help():
|
| 8 |
-
from app_cli import handle_command
|
| 9 |
-
result = handle_command("/help", cfg={}, state={})
|
| 10 |
-
assert result is not None
|
| 11 |
-
assert "quit" in result.lower() or "help" in result.lower()
|
| 12 |
-
|
| 13 |
-
def test_handle_command_unknown():
|
| 14 |
-
from app_cli import handle_command
|
| 15 |
-
result = handle_command("What is nonviolent resistance?", cfg={}, state={})
|
| 16 |
-
assert result is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_config_loader.py
DELETED
|
@@ -1,135 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pytest
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
def test_load_config_from_yaml(tmp_path):
|
| 6 |
-
yaml_content = """
|
| 7 |
-
chatbot:
|
| 8 |
-
name: "Test Bot"
|
| 9 |
-
domain: "testing"
|
| 10 |
-
llm:
|
| 11 |
-
provider: "openai"
|
| 12 |
-
model: "gpt-4o"
|
| 13 |
-
temperature: 0.0
|
| 14 |
-
max_tokens: 2048
|
| 15 |
-
api_keys:
|
| 16 |
-
openai: ""
|
| 17 |
-
anthropic: ""
|
| 18 |
-
gemini: ""
|
| 19 |
-
embeddings:
|
| 20 |
-
provider: "local"
|
| 21 |
-
openai_model: "text-embedding-3-small"
|
| 22 |
-
retrieval:
|
| 23 |
-
chunk_size: 1000
|
| 24 |
-
chunk_overlap: 100
|
| 25 |
-
top_k: 5
|
| 26 |
-
web_search:
|
| 27 |
-
enabled: true
|
| 28 |
-
backend: "semantic_scholar"
|
| 29 |
-
max_results: 5
|
| 30 |
-
verification:
|
| 31 |
-
enabled: true
|
| 32 |
-
max_iterations: 3
|
| 33 |
-
strict_mode: true
|
| 34 |
-
paths:
|
| 35 |
-
knowledge_base: "knowledge_base"
|
| 36 |
-
vector_db: "chroma_db"
|
| 37 |
-
"""
|
| 38 |
-
config_file = tmp_path / "config.yaml"
|
| 39 |
-
config_file.write_text(yaml_content)
|
| 40 |
-
|
| 41 |
-
from src.config_loader import load_config
|
| 42 |
-
cfg = load_config(str(config_file))
|
| 43 |
-
|
| 44 |
-
assert cfg["chatbot"]["name"] == "Test Bot"
|
| 45 |
-
assert cfg["llm"]["provider"] == "openai"
|
| 46 |
-
assert cfg["llm"]["model"] == "gpt-4o"
|
| 47 |
-
assert cfg["retrieval"]["top_k"] == 5
|
| 48 |
-
assert cfg["verification"]["enabled"] is True
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def test_env_vars_override_api_keys(tmp_path):
|
| 52 |
-
yaml_content = """
|
| 53 |
-
chatbot:
|
| 54 |
-
name: "Test"
|
| 55 |
-
domain: "test"
|
| 56 |
-
llm:
|
| 57 |
-
provider: "openai"
|
| 58 |
-
model: "gpt-4o"
|
| 59 |
-
temperature: 0.0
|
| 60 |
-
max_tokens: 2048
|
| 61 |
-
api_keys:
|
| 62 |
-
openai: "yaml-key"
|
| 63 |
-
anthropic: ""
|
| 64 |
-
gemini: ""
|
| 65 |
-
embeddings:
|
| 66 |
-
provider: "local"
|
| 67 |
-
openai_model: "text-embedding-3-small"
|
| 68 |
-
retrieval:
|
| 69 |
-
chunk_size: 1000
|
| 70 |
-
chunk_overlap: 100
|
| 71 |
-
top_k: 5
|
| 72 |
-
web_search:
|
| 73 |
-
enabled: false
|
| 74 |
-
backend: "semantic_scholar"
|
| 75 |
-
max_results: 5
|
| 76 |
-
verification:
|
| 77 |
-
enabled: true
|
| 78 |
-
max_iterations: 3
|
| 79 |
-
strict_mode: true
|
| 80 |
-
paths:
|
| 81 |
-
knowledge_base: "knowledge_base"
|
| 82 |
-
vector_db: "chroma_db"
|
| 83 |
-
"""
|
| 84 |
-
config_file = tmp_path / "config.yaml"
|
| 85 |
-
config_file.write_text(yaml_content)
|
| 86 |
-
|
| 87 |
-
os.environ["OPENAI_API_KEY"] = "env-key"
|
| 88 |
-
try:
|
| 89 |
-
from src.config_loader import load_config
|
| 90 |
-
cfg = load_config(str(config_file))
|
| 91 |
-
assert cfg["api_keys"]["openai"] == "env-key"
|
| 92 |
-
finally:
|
| 93 |
-
del os.environ["OPENAI_API_KEY"]
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
def test_get_api_key_helper(tmp_path):
|
| 97 |
-
yaml_content = """
|
| 98 |
-
chatbot:
|
| 99 |
-
name: "Test"
|
| 100 |
-
domain: "test"
|
| 101 |
-
llm:
|
| 102 |
-
provider: "openai"
|
| 103 |
-
model: "gpt-4o"
|
| 104 |
-
temperature: 0.0
|
| 105 |
-
max_tokens: 2048
|
| 106 |
-
api_keys:
|
| 107 |
-
openai: "test-key-123"
|
| 108 |
-
anthropic: ""
|
| 109 |
-
gemini: ""
|
| 110 |
-
embeddings:
|
| 111 |
-
provider: "local"
|
| 112 |
-
openai_model: "text-embedding-3-small"
|
| 113 |
-
retrieval:
|
| 114 |
-
chunk_size: 1000
|
| 115 |
-
chunk_overlap: 100
|
| 116 |
-
top_k: 5
|
| 117 |
-
web_search:
|
| 118 |
-
enabled: false
|
| 119 |
-
backend: "semantic_scholar"
|
| 120 |
-
max_results: 5
|
| 121 |
-
verification:
|
| 122 |
-
enabled: true
|
| 123 |
-
max_iterations: 3
|
| 124 |
-
strict_mode: true
|
| 125 |
-
paths:
|
| 126 |
-
knowledge_base: "knowledge_base"
|
| 127 |
-
vector_db: "chroma_db"
|
| 128 |
-
"""
|
| 129 |
-
config_file = tmp_path / "config.yaml"
|
| 130 |
-
config_file.write_text(yaml_content)
|
| 131 |
-
|
| 132 |
-
from src.config_loader import load_config, get_api_key
|
| 133 |
-
cfg = load_config(str(config_file))
|
| 134 |
-
assert get_api_key(cfg, "openai") == "test-key-123"
|
| 135 |
-
assert get_api_key(cfg, "anthropic") == ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_ingest.py
DELETED
|
@@ -1,53 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
def test_split_text_recursive_short():
|
| 5 |
-
"""Short text returns as single chunk."""
|
| 6 |
-
from src.ingest import split_text_recursive
|
| 7 |
-
chunks = split_text_recursive("Hello world", 1000, 100)
|
| 8 |
-
assert chunks == ["Hello world"]
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def test_split_text_recursive_long():
|
| 12 |
-
"""Long text is split into overlapping chunks."""
|
| 13 |
-
from src.ingest import split_text_recursive
|
| 14 |
-
text = "Paragraph one. " * 100 + "\n\n" + "Paragraph two. " * 100
|
| 15 |
-
chunks = split_text_recursive(text, 500, 50)
|
| 16 |
-
assert len(chunks) > 1
|
| 17 |
-
for chunk in chunks:
|
| 18 |
-
assert len(chunk) <= 600 # allow some overflow at boundaries
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
def test_chunk_documents():
|
| 22 |
-
"""Pages are chunked with correct metadata."""
|
| 23 |
-
from src.ingest import chunk_documents
|
| 24 |
-
pages = [{"page": 1, "text": "A short page of text."}]
|
| 25 |
-
chunks = chunk_documents(pages, "test.pdf", "general")
|
| 26 |
-
assert len(chunks) >= 1
|
| 27 |
-
assert chunks[0]["metadata"]["source"] == "test.pdf"
|
| 28 |
-
assert chunks[0]["metadata"]["dataset"] == "general"
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
def test_discover_files(tmp_path):
|
| 32 |
-
"""discover_files finds supported files recursively."""
|
| 33 |
-
from src.ingest import discover_files
|
| 34 |
-
|
| 35 |
-
# Create test files
|
| 36 |
-
sub = tmp_path / "Dataset1"
|
| 37 |
-
sub.mkdir()
|
| 38 |
-
(sub / "paper.pdf").write_bytes(b"%PDF-1.4 fake")
|
| 39 |
-
(sub / "data.csv").write_text("a,b\n1,2\n")
|
| 40 |
-
(tmp_path / "notes.txt").write_text("hello")
|
| 41 |
-
(tmp_path / "photo.jpg").write_bytes(b"fake jpg") # unsupported
|
| 42 |
-
|
| 43 |
-
files = discover_files(str(tmp_path))
|
| 44 |
-
extensions = {f[0].suffix for f in files}
|
| 45 |
-
assert ".pdf" in extensions
|
| 46 |
-
assert ".csv" in extensions
|
| 47 |
-
assert ".txt" in extensions
|
| 48 |
-
assert ".jpg" not in extensions
|
| 49 |
-
|
| 50 |
-
# Check dataset names
|
| 51 |
-
datasets = {f[1] for f in files}
|
| 52 |
-
assert "Dataset1" in datasets
|
| 53 |
-
assert "general" in datasets
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_kb_meta.py
DELETED
|
@@ -1,255 +0,0 @@
|
|
| 1 |
-
"""Tests for src.kb_meta — KB overview generation and storage."""
|
| 2 |
-
|
| 3 |
-
import json
|
| 4 |
-
import os
|
| 5 |
-
from unittest.mock import patch
|
| 6 |
-
|
| 7 |
-
import pytest
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
# ── Deterministic overview tests ─────────────────────────────────────────────
|
| 11 |
-
|
| 12 |
-
def test_generate_kb_overview_empty():
|
| 13 |
-
from src.kb_meta import generate_kb_overview
|
| 14 |
-
assert generate_kb_overview([], {}, None) == ""
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
def test_generate_kb_overview_basic():
|
| 18 |
-
from src.kb_meta import generate_kb_overview
|
| 19 |
-
records = [
|
| 20 |
-
{"source": "ds1/file1.pdf", "dataset": "ds1", "ext": ".pdf", "chunk_count": 10},
|
| 21 |
-
{"source": "ds1/file2.csv", "dataset": "ds1", "ext": ".csv", "chunk_count": 5},
|
| 22 |
-
{"source": "general/readme.txt", "dataset": "general", "ext": ".txt", "chunk_count": 2},
|
| 23 |
-
]
|
| 24 |
-
overview = generate_kb_overview(records, {}, None)
|
| 25 |
-
assert "KNOWLEDGE BASE OVERVIEW" in overview
|
| 26 |
-
assert "3 files" in overview
|
| 27 |
-
assert "17 chunks" in overview
|
| 28 |
-
assert "[ds1]" in overview
|
| 29 |
-
assert "file1.pdf" in overview
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
def test_generate_kb_overview_with_sql():
|
| 33 |
-
from src.kb_meta import generate_kb_overview
|
| 34 |
-
records = [{"source": "d/f.csv", "dataset": "d", "ext": ".csv", "chunk_count": 3}]
|
| 35 |
-
schema = {
|
| 36 |
-
"d__f_csv": {
|
| 37 |
-
"columns": [{"name": "x", "type": "TEXT"}, {"name": "y", "type": "INTEGER"}],
|
| 38 |
-
"row_count": 100,
|
| 39 |
-
"source_file": "d/f.csv",
|
| 40 |
-
}
|
| 41 |
-
}
|
| 42 |
-
overview = generate_kb_overview(records, {}, schema)
|
| 43 |
-
assert "SQL table" in overview
|
| 44 |
-
assert "100 rows" in overview
|
| 45 |
-
assert "x, y" in overview
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
def test_generate_kb_overview_multi_dataset_connections():
|
| 49 |
-
from src.kb_meta import generate_kb_overview
|
| 50 |
-
records = [
|
| 51 |
-
{"source": "a/f1.pdf", "dataset": "a", "ext": ".pdf", "chunk_count": 5},
|
| 52 |
-
{"source": "b/f2.pdf", "dataset": "b", "ext": ".pdf", "chunk_count": 3},
|
| 53 |
-
]
|
| 54 |
-
overview = generate_kb_overview(records, {}, None)
|
| 55 |
-
# With multiple datasets, lists both
|
| 56 |
-
assert "[a]" in overview
|
| 57 |
-
assert "[b]" in overview
|
| 58 |
-
assert "2 dataset" in overview
|
| 59 |
-
# Connection note present for multi-dataset
|
| 60 |
-
assert "same knowledge base" in overview
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
# ── File record collection ───────────────────────────────────────────────────
|
| 64 |
-
|
| 65 |
-
def test_collect_file_records_empty(tmp_path):
|
| 66 |
-
"""collect_file_records returns [] for an empty collection."""
|
| 67 |
-
import chromadb
|
| 68 |
-
client = chromadb.Client()
|
| 69 |
-
col = client.get_or_create_collection("test_empty")
|
| 70 |
-
from src.kb_meta import collect_file_records
|
| 71 |
-
assert collect_file_records(col) == []
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
def test_collect_file_records_groups_by_source(tmp_path):
|
| 75 |
-
"""collect_file_records aggregates chunk counts per source."""
|
| 76 |
-
import chromadb
|
| 77 |
-
client = chromadb.Client()
|
| 78 |
-
col = client.get_or_create_collection("test_records")
|
| 79 |
-
col.add(
|
| 80 |
-
ids=["a1", "a2", "b1"],
|
| 81 |
-
documents=["chunk1", "chunk2", "chunk3"],
|
| 82 |
-
metadatas=[
|
| 83 |
-
{"source": "ds/file_a.pdf", "dataset": "ds", "page": "1"},
|
| 84 |
-
{"source": "ds/file_a.pdf", "dataset": "ds", "page": "2"},
|
| 85 |
-
{"source": "ds/file_b.csv", "dataset": "ds", "page": "1"},
|
| 86 |
-
],
|
| 87 |
-
)
|
| 88 |
-
from src.kb_meta import collect_file_records
|
| 89 |
-
records = collect_file_records(col)
|
| 90 |
-
assert len(records) == 2
|
| 91 |
-
a_rec = next(r for r in records if "file_a" in r["source"])
|
| 92 |
-
assert a_rec["chunk_count"] == 2
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
# ── Sample chunk collection ──────────────────────────────────────────────────
|
| 96 |
-
|
| 97 |
-
def test_collect_sample_chunks():
|
| 98 |
-
import chromadb
|
| 99 |
-
client = chromadb.Client()
|
| 100 |
-
col = client.get_or_create_collection("test_samples")
|
| 101 |
-
col.add(
|
| 102 |
-
ids=["x1", "x2"],
|
| 103 |
-
documents=["first chunk of A", "second chunk of A"],
|
| 104 |
-
metadatas=[
|
| 105 |
-
{"source": "A.pdf", "dataset": "d", "page": "1"},
|
| 106 |
-
{"source": "A.pdf", "dataset": "d", "page": "2"},
|
| 107 |
-
],
|
| 108 |
-
)
|
| 109 |
-
from src.kb_meta import collect_sample_chunks
|
| 110 |
-
samples = collect_sample_chunks(col)
|
| 111 |
-
assert "A.pdf" in samples
|
| 112 |
-
assert samples["A.pdf"] == "first chunk of A"
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
# ── Meta chunk upsert ────────────────────────────────────────────────────────
|
| 116 |
-
|
| 117 |
-
def test_upsert_meta_chunk():
|
| 118 |
-
import chromadb
|
| 119 |
-
from src.kb_meta import upsert_meta_chunk, META_CHUNK_ID
|
| 120 |
-
client = chromadb.Client()
|
| 121 |
-
col = client.get_or_create_collection("test_upsert")
|
| 122 |
-
upsert_meta_chunk(col, "test overview text")
|
| 123 |
-
result = col.get(ids=[META_CHUNK_ID])
|
| 124 |
-
assert result["documents"][0] == "test overview text"
|
| 125 |
-
|
| 126 |
-
# Upsert again — should update, not duplicate
|
| 127 |
-
upsert_meta_chunk(col, "updated overview")
|
| 128 |
-
assert col.count() == 1
|
| 129 |
-
result = col.get(ids=[META_CHUNK_ID])
|
| 130 |
-
assert result["documents"][0] == "updated overview"
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
# ── Save and load ────────────────────────────────────────────────────────────
|
| 134 |
-
|
| 135 |
-
def test_save_and_load_kb_meta(tmp_path):
|
| 136 |
-
from src.kb_meta import save_kb_meta, load_kb_meta
|
| 137 |
-
cfg = {"paths": {"vector_db": str(tmp_path / "chroma_db")}}
|
| 138 |
-
save_kb_meta("my overview", cfg)
|
| 139 |
-
loaded = load_kb_meta(cfg)
|
| 140 |
-
assert loaded == "my overview"
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
def test_load_kb_meta_missing(tmp_path):
|
| 144 |
-
from src.kb_meta import load_kb_meta
|
| 145 |
-
cfg = {"paths": {"vector_db": str(tmp_path / "nonexistent")}}
|
| 146 |
-
assert load_kb_meta(cfg) == ""
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
# ── Prompt integration ───────────────────────────────────────────────────────
|
| 150 |
-
|
| 151 |
-
def test_build_prompt_includes_kb_overview():
|
| 152 |
-
from src.prompts import build_prompt
|
| 153 |
-
prompt = build_prompt("context", "Bot", "research", kb_overview="MY_KB_OVERVIEW")
|
| 154 |
-
assert "MY_KB_OVERVIEW" in prompt
|
| 155 |
-
assert "KNOWLEDGE BASE OVERVIEW" in prompt
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
def test_build_prompt_no_overview():
|
| 159 |
-
from src.prompts import build_prompt
|
| 160 |
-
prompt = build_prompt("context", "Bot", "research", kb_overview="")
|
| 161 |
-
assert "KNOWLEDGE BASE OVERVIEW" not in prompt
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
def test_qu_prompt_includes_kb_overview():
|
| 165 |
-
from src.prompts import build_query_understanding_prompt
|
| 166 |
-
prompt = build_query_understanding_prompt(
|
| 167 |
-
"test query", "research", [],
|
| 168 |
-
kb_overview="OVERVIEW_TEXT_HERE",
|
| 169 |
-
)
|
| 170 |
-
assert "OVERVIEW_TEXT_HERE" in prompt
|
| 171 |
-
assert "KNOWLEDGE BASE CONTENTS" in prompt
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
def test_qu_prompt_no_overview():
|
| 175 |
-
from src.prompts import build_query_understanding_prompt
|
| 176 |
-
prompt = build_query_understanding_prompt("test query", "research", [])
|
| 177 |
-
assert "KNOWLEDGE BASE CONTENTS" not in prompt
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
# ── LLM overview generation ──────────────────────────────────────────────────
|
| 181 |
-
|
| 182 |
-
def _mock_llm_generate(system_prompt, user_message, cfg, **kwargs):
|
| 183 |
-
"""Mock LLM that returns a KB overview."""
|
| 184 |
-
return "## Overview\nThis KB covers human rights data across 2 datasets."
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
def _mock_llm_generate_fail(system_prompt, user_message, cfg, **kwargs):
|
| 188 |
-
"""Mock LLM that raises an exception."""
|
| 189 |
-
raise RuntimeError("LLM unavailable")
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
def test_generate_kb_overview_with_llm_success():
|
| 193 |
-
from src.kb_meta import generate_kb_overview_with_llm
|
| 194 |
-
records = [{"source": "d/f.pdf", "dataset": "d", "ext": ".pdf", "chunk_count": 5}]
|
| 195 |
-
samples = {"d/f.pdf": "Introduction to human rights..."}
|
| 196 |
-
cfg = {"llm": {"provider": "openai", "model": "gpt-4o"}, "api_keys": {"openai": "fake"}}
|
| 197 |
-
with patch("src.llm.generate", _mock_llm_generate):
|
| 198 |
-
overview = generate_kb_overview_with_llm(records, samples, None, cfg)
|
| 199 |
-
assert "KNOWLEDGE BASE OVERVIEW" in overview
|
| 200 |
-
assert "human rights" in overview
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
def test_generate_kb_overview_with_llm_fallback():
|
| 204 |
-
from src.kb_meta import generate_kb_overview_with_llm
|
| 205 |
-
records = [{"source": "d/f.pdf", "dataset": "d", "ext": ".pdf", "chunk_count": 5}]
|
| 206 |
-
samples = {"d/f.pdf": "text"}
|
| 207 |
-
cfg = {"llm": {"provider": "openai", "model": "gpt-4o"}, "api_keys": {"openai": "fake"}}
|
| 208 |
-
with patch("src.llm.generate", _mock_llm_generate_fail):
|
| 209 |
-
overview = generate_kb_overview_with_llm(records, samples, None, cfg)
|
| 210 |
-
# Falls back to deterministic overview
|
| 211 |
-
assert "KNOWLEDGE BASE OVERVIEW" in overview
|
| 212 |
-
assert "d/f.pdf" in overview
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
def test_generate_kb_overview_with_llm_empty():
|
| 216 |
-
from src.kb_meta import generate_kb_overview_with_llm
|
| 217 |
-
cfg = {"llm": {}}
|
| 218 |
-
result = generate_kb_overview_with_llm([], {}, None, cfg)
|
| 219 |
-
assert result == ""
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
# ── Brace safety (format injection regression) ──────────────────────────────
|
| 223 |
-
|
| 224 |
-
def test_llm_overview_survives_braces_in_samples():
|
| 225 |
-
"""Sample chunks with {braces} must not crash the prompt formatter."""
|
| 226 |
-
from src.kb_meta import generate_kb_overview_with_llm
|
| 227 |
-
records = [{"source": "d/code.py", "dataset": "d", "ext": ".py", "chunk_count": 1}]
|
| 228 |
-
samples = {"d/code.py": 'def foo(): return {"key": value}'}
|
| 229 |
-
cfg = {"llm": {"provider": "openai", "model": "gpt-4o"}, "api_keys": {"openai": "fake"}}
|
| 230 |
-
with patch("src.llm.generate", _mock_llm_generate):
|
| 231 |
-
# Should not raise KeyError or ValueError
|
| 232 |
-
overview = generate_kb_overview_with_llm(records, samples, None, cfg)
|
| 233 |
-
assert "KNOWLEDGE BASE OVERVIEW" in overview
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
def test_build_prompt_kb_overview_with_braces():
|
| 237 |
-
"""KB overview containing {braces} must not crash the system prompt builder."""
|
| 238 |
-
from src.prompts import build_prompt
|
| 239 |
-
overview = "Dataset contains {variable_name} and LaTeX {R}."
|
| 240 |
-
# Should not raise — braces are escaped by build_prompt
|
| 241 |
-
prompt = build_prompt("context", "Bot", "research", kb_overview=overview)
|
| 242 |
-
# Braces in substituted values stay doubled after str.format() (pre-existing behavior)
|
| 243 |
-
assert "variable_name" in prompt
|
| 244 |
-
assert "KNOWLEDGE BASE OVERVIEW" in prompt
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
def test_qu_prompt_kb_overview_with_braces():
|
| 248 |
-
"""KB overview containing {braces} must not crash the QU prompt builder."""
|
| 249 |
-
from src.prompts import build_query_understanding_prompt
|
| 250 |
-
overview = "Table has columns {id}, {name}, {value}."
|
| 251 |
-
prompt = build_query_understanding_prompt(
|
| 252 |
-
"test", "research", [], kb_overview=overview,
|
| 253 |
-
)
|
| 254 |
-
assert "{id}" in prompt
|
| 255 |
-
assert "{name}" in prompt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_llm.py
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_provider_registry_has_all_providers():
|
| 4 |
-
from src.llm import PROVIDERS
|
| 5 |
-
assert "openai" in PROVIDERS
|
| 6 |
-
assert "anthropic" in PROVIDERS
|
| 7 |
-
assert "gemini" in PROVIDERS
|
| 8 |
-
|
| 9 |
-
def test_generate_raises_on_unknown_provider():
|
| 10 |
-
from src.llm import generate
|
| 11 |
-
with pytest.raises(ValueError, match="Unknown provider"):
|
| 12 |
-
generate("system", "user", provider="fake", cfg={"api_keys": {}})
|
| 13 |
-
|
| 14 |
-
def test_generate_raises_on_missing_api_key():
|
| 15 |
-
from src.llm import generate
|
| 16 |
-
cfg = {"api_keys": {"openai": ""}, "llm": {"provider": "openai", "model": "gpt-4o",
|
| 17 |
-
"temperature": 0.0, "max_tokens": 100}}
|
| 18 |
-
with pytest.raises(ValueError, match="API key"):
|
| 19 |
-
generate("system", "user", provider="openai", cfg=cfg)
|
| 20 |
-
|
| 21 |
-
def test_list_models_fallback():
|
| 22 |
-
from src.llm.openai import list_models
|
| 23 |
-
models = list_models("invalid-key-123")
|
| 24 |
-
assert isinstance(models, list)
|
| 25 |
-
assert len(models) > 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_prompts.py
DELETED
|
@@ -1,111 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
def test_build_prompt_includes_anti_hallucination():
|
| 5 |
-
from src.prompts import build_prompt
|
| 6 |
-
prompt = build_prompt("some context", "Test Bot", "testing domain")
|
| 7 |
-
assert "ZERO TOLERANCE" in prompt
|
| 8 |
-
assert "NEVER use your training data" in prompt
|
| 9 |
-
assert "refuse" in prompt.lower()
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
def test_build_prompt_includes_citation_rules():
|
| 13 |
-
from src.prompts import build_prompt
|
| 14 |
-
prompt = build_prompt("some context", "Test Bot", "testing")
|
| 15 |
-
assert "endnote" in prompt.lower()
|
| 16 |
-
assert "References" in prompt
|
| 17 |
-
assert "direct quote" in prompt.lower() or "Direct quote" in prompt
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def test_build_prompt_includes_context():
|
| 21 |
-
from src.prompts import build_prompt
|
| 22 |
-
prompt = build_prompt("MY_UNIQUE_CONTEXT_STRING", "Bot", "domain")
|
| 23 |
-
assert "MY_UNIQUE_CONTEXT_STRING" in prompt
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
def test_build_prompt_includes_bot_identity():
|
| 27 |
-
from src.prompts import build_prompt
|
| 28 |
-
prompt = build_prompt("ctx", "ResearchHelper", "political science")
|
| 29 |
-
assert "ResearchHelper" in prompt
|
| 30 |
-
assert "political science" in prompt
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def test_build_verification_prompt():
|
| 34 |
-
from src.prompts import build_verification_prompt
|
| 35 |
-
prompt = build_verification_prompt("response text", "context text", [], [])
|
| 36 |
-
assert "citation" in prompt.lower()
|
| 37 |
-
assert "JSON" in prompt or "json" in prompt
|
| 38 |
-
assert "error_count" in prompt
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
def test_build_query_understanding_prompt():
|
| 42 |
-
from src.prompts import build_query_understanding_prompt
|
| 43 |
-
prompt = build_query_understanding_prompt(
|
| 44 |
-
"What happened in China?",
|
| 45 |
-
"human rights",
|
| 46 |
-
[{"role": "user", "content": "prior question"}],
|
| 47 |
-
)
|
| 48 |
-
assert "human rights" in prompt
|
| 49 |
-
assert "What happened in China?" in prompt
|
| 50 |
-
assert "prior question" in prompt
|
| 51 |
-
assert "SEARCH" in prompt or "search" in prompt
|
| 52 |
-
assert "CLARIFY" in prompt or "clarify" in prompt
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def test_build_query_understanding_prompt_no_history():
|
| 56 |
-
from src.prompts import build_query_understanding_prompt
|
| 57 |
-
prompt = build_query_understanding_prompt("test query", "research", [])
|
| 58 |
-
assert "No prior conversation" in prompt
|
| 59 |
-
assert "test query" in prompt
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
def test_build_prompt_with_curly_braces_in_context():
|
| 63 |
-
"""Regression: context containing {placeholder} must not crash str.format()."""
|
| 64 |
-
from src.prompts import build_prompt
|
| 65 |
-
context = "The variable {unknown_var} was set to {value}."
|
| 66 |
-
prompt = build_prompt(context, "Bot", "testing")
|
| 67 |
-
assert "{unknown_var}" in prompt
|
| 68 |
-
assert "{value}" in prompt
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
def test_build_verification_prompt_with_curly_braces():
|
| 72 |
-
"""Regression: braces in response/context must not crash str.format()."""
|
| 73 |
-
from src.prompts import build_verification_prompt
|
| 74 |
-
prompt = build_verification_prompt(
|
| 75 |
-
"The {result} showed improvement.",
|
| 76 |
-
"Data: {raw_value} was recorded.",
|
| 77 |
-
["flag with {braces}"],
|
| 78 |
-
[],
|
| 79 |
-
)
|
| 80 |
-
assert "{result}" in prompt
|
| 81 |
-
assert "{raw_value}" in prompt
|
| 82 |
-
assert "{braces}" in prompt
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
def test_qu_prompt_includes_schema_when_provided():
|
| 86 |
-
from src.prompts import build_query_understanding_prompt
|
| 87 |
-
schema_summary = "Available SQL tables:\n- t (100 rows): x (TEXT)"
|
| 88 |
-
prompt = build_query_understanding_prompt(
|
| 89 |
-
"test query", "research", [], sql_schema_summary=schema_summary,
|
| 90 |
-
)
|
| 91 |
-
assert "Available SQL tables" in prompt
|
| 92 |
-
assert "route" in prompt
|
| 93 |
-
assert "sql_query" in prompt
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
def test_qu_prompt_no_schema_no_sql_fields():
|
| 97 |
-
from src.prompts import build_query_understanding_prompt
|
| 98 |
-
prompt = build_query_understanding_prompt("test query", "research", [])
|
| 99 |
-
assert "Available SQL tables" not in prompt
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
def test_system_prompt_mentions_chunk_sql():
|
| 103 |
-
from src.prompts import build_prompt
|
| 104 |
-
prompt = build_prompt("context", "Bot", "research")
|
| 105 |
-
assert "CHUNK-SQL" in prompt
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
def test_verification_prompt_sql_priority():
|
| 109 |
-
from src.prompts import build_verification_prompt
|
| 110 |
-
prompt = build_verification_prompt("resp", "ctx", [], [])
|
| 111 |
-
assert "CHUNK-SQL" in prompt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_query_engine.py
DELETED
|
@@ -1,172 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
import pytest
|
| 3 |
-
from unittest.mock import patch
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
def _make_cfg(enabled=True, max_history=6, max_clarifications=1):
|
| 7 |
-
"""Build a minimal config dict for query engine tests."""
|
| 8 |
-
return {
|
| 9 |
-
"chatbot": {"name": "TestBot", "domain": "human rights research"},
|
| 10 |
-
"llm": {"provider": "openai", "model": "gpt-4o", "temperature": 0.0, "max_tokens": 2048},
|
| 11 |
-
"api_keys": {"openai": "fake-key"},
|
| 12 |
-
"query_understanding": {
|
| 13 |
-
"enabled": enabled,
|
| 14 |
-
"max_history": max_history,
|
| 15 |
-
"max_clarifications": max_clarifications,
|
| 16 |
-
},
|
| 17 |
-
}
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def _mock_generate_search(system_prompt, user_message, cfg, **kwargs):
|
| 21 |
-
"""Mock LLM that always returns a search reformulation."""
|
| 22 |
-
return json.dumps({
|
| 23 |
-
"action": "search",
|
| 24 |
-
"search_query": "reformulated test query about human rights",
|
| 25 |
-
"display_query": "What are the key human rights issues in this test query?",
|
| 26 |
-
"reasoning": "expanded for better retrieval",
|
| 27 |
-
})
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
def _mock_generate_clarify(system_prompt, user_message, cfg, **kwargs):
|
| 31 |
-
"""Mock LLM that always returns a clarification request."""
|
| 32 |
-
return json.dumps({
|
| 33 |
-
"action": "clarify",
|
| 34 |
-
"clarification_question": "Are you asking about X or Y?",
|
| 35 |
-
"reasoning": "ambiguous query",
|
| 36 |
-
})
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
def _mock_generate_error(system_prompt, user_message, cfg, **kwargs):
|
| 40 |
-
"""Mock LLM that raises an exception."""
|
| 41 |
-
raise RuntimeError("LLM unavailable")
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
def test_understand_query_reformulates():
|
| 45 |
-
from src.query_engine import understand_query
|
| 46 |
-
with patch("src.query_engine.generate", _mock_generate_search):
|
| 47 |
-
result = understand_query("test query", _make_cfg())
|
| 48 |
-
assert result["action"] == "search"
|
| 49 |
-
assert result["search_query"] == "reformulated test query about human rights"
|
| 50 |
-
assert result["display_query"] == "What are the key human rights issues in this test query?"
|
| 51 |
-
assert result["original_query"] == "test query"
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def test_understand_query_clarifies():
|
| 55 |
-
from src.query_engine import understand_query
|
| 56 |
-
with patch("src.query_engine.generate", _mock_generate_clarify):
|
| 57 |
-
result = understand_query("AI", _make_cfg())
|
| 58 |
-
assert result["action"] == "clarify"
|
| 59 |
-
assert "clarification_question" in result
|
| 60 |
-
assert result["original_query"] == "AI"
|
| 61 |
-
# search_query and display_query should still be present as fallback
|
| 62 |
-
assert "search_query" in result
|
| 63 |
-
assert "display_query" in result
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
def test_understand_query_preserves_original():
|
| 67 |
-
from src.query_engine import understand_query
|
| 68 |
-
original = "What happened in China?"
|
| 69 |
-
with patch("src.query_engine.generate", _mock_generate_search):
|
| 70 |
-
result = understand_query(original, _make_cfg())
|
| 71 |
-
assert result["original_query"] == original
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
def test_understand_query_with_history():
|
| 75 |
-
from src.query_engine import understand_query
|
| 76 |
-
history = [
|
| 77 |
-
{"role": "user", "content": "Tell me about China"},
|
| 78 |
-
{"role": "assistant", "content": "China has various human rights issues..."},
|
| 79 |
-
]
|
| 80 |
-
with patch("src.query_engine.generate", _mock_generate_search):
|
| 81 |
-
result = understand_query("what about 2023?", _make_cfg(), history)
|
| 82 |
-
assert result["action"] == "search"
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
def test_understand_query_disabled():
|
| 86 |
-
from src.query_engine import understand_query
|
| 87 |
-
result = understand_query("test query", _make_cfg(enabled=False))
|
| 88 |
-
assert result["action"] == "search"
|
| 89 |
-
assert result["search_query"] == "test query"
|
| 90 |
-
assert result["display_query"] == "test query"
|
| 91 |
-
assert result["original_query"] == "test query"
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def test_understand_query_fallback_on_error():
|
| 95 |
-
from src.query_engine import understand_query
|
| 96 |
-
with patch("src.query_engine.generate", _mock_generate_error):
|
| 97 |
-
result = understand_query("test query", _make_cfg())
|
| 98 |
-
assert result["action"] == "search"
|
| 99 |
-
assert result["search_query"] == "test query"
|
| 100 |
-
assert result["display_query"] == "test query"
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
def test_parse_qu_result_malformed_json():
|
| 104 |
-
from src.query_engine import _parse_qu_result
|
| 105 |
-
result = _parse_qu_result("not json at all", "original query")
|
| 106 |
-
assert result["action"] == "search"
|
| 107 |
-
assert result["search_query"] == "original query"
|
| 108 |
-
assert result["display_query"] == "original query"
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
def test_parse_qu_result_embedded_json():
|
| 112 |
-
from src.query_engine import _parse_qu_result
|
| 113 |
-
raw = 'Here is the result: {"action": "search", "search_query": "better query", "reasoning": "test"} end'
|
| 114 |
-
result = _parse_qu_result(raw, "original")
|
| 115 |
-
assert result["action"] == "search"
|
| 116 |
-
assert result["search_query"] == "better query"
|
| 117 |
-
# display_query falls back to original when not in JSON
|
| 118 |
-
assert result["display_query"] == "original"
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def test_parse_qu_result_with_display_query():
|
| 122 |
-
from src.query_engine import _parse_qu_result
|
| 123 |
-
raw = json.dumps({
|
| 124 |
-
"action": "search",
|
| 125 |
-
"search_query": "keyword optimized query",
|
| 126 |
-
"display_query": "What is the effect of AI on human rights?",
|
| 127 |
-
"reasoning": "clarified and expanded",
|
| 128 |
-
})
|
| 129 |
-
result = _parse_qu_result(raw, "AI human rights")
|
| 130 |
-
assert result["search_query"] == "keyword optimized query"
|
| 131 |
-
assert result["display_query"] == "What is the effect of AI on human rights?"
|
| 132 |
-
assert result["original_query"] == "AI human rights"
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
def test_parse_qu_result_with_route_and_sql():
|
| 136 |
-
from src.query_engine import _parse_qu_result
|
| 137 |
-
raw = '{"action": "search", "route": "sql", "search_query": "PTS China", "display_query": "PTS scores for China?", "sql_query": "SELECT * FROM t", "reasoning": "data lookup"}'
|
| 138 |
-
result = _parse_qu_result(raw, "original")
|
| 139 |
-
assert result["route"] == "sql"
|
| 140 |
-
assert result["sql_query"] == "SELECT * FROM t"
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
def test_parse_qu_result_defaults_route_to_vector():
|
| 144 |
-
from src.query_engine import _parse_qu_result
|
| 145 |
-
raw = '{"action": "search", "search_query": "test", "display_query": "test?"}'
|
| 146 |
-
result = _parse_qu_result(raw, "original")
|
| 147 |
-
assert result["route"] == "vector"
|
| 148 |
-
assert result.get("sql_query") is None
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
def test_load_sql_schema_missing_file():
|
| 152 |
-
from src.query_engine import _load_sql_schema_summary
|
| 153 |
-
result = _load_sql_schema_summary("/nonexistent/path/sql_db")
|
| 154 |
-
assert result == ""
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
def test_load_sql_schema_valid_file(tmp_path):
|
| 158 |
-
import json
|
| 159 |
-
from src.query_engine import _load_sql_schema_summary
|
| 160 |
-
schema_dir = tmp_path / "sql_db"
|
| 161 |
-
schema_dir.mkdir()
|
| 162 |
-
schema = {
|
| 163 |
-
"test_table": {
|
| 164 |
-
"source_file": "data.csv",
|
| 165 |
-
"columns": [{"name": "x", "type": "TEXT", "sample": ["a"]}],
|
| 166 |
-
"row_count": 10,
|
| 167 |
-
}
|
| 168 |
-
}
|
| 169 |
-
(schema_dir / "sql_schemas.json").write_text(json.dumps(schema))
|
| 170 |
-
result = _load_sql_schema_summary(str(schema_dir))
|
| 171 |
-
assert "test_table" in result
|
| 172 |
-
assert "10 rows" in result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_readers.py
DELETED
|
@@ -1,47 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
def test_reader_registry_has_all_extensions():
|
| 6 |
-
from src.readers import READERS
|
| 7 |
-
expected = {
|
| 8 |
-
".pdf", ".docx", ".xlsx", ".xls", ".csv", ".tab", ".tsv",
|
| 9 |
-
".dta", ".sav", ".rds", ".rda", ".txt", ".md", ".json", ".do",
|
| 10 |
-
}
|
| 11 |
-
assert expected == set(READERS.keys())
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
def test_text_reader_txt(tmp_path):
|
| 15 |
-
f = tmp_path / "notes.txt"
|
| 16 |
-
f.write_text("This is a test document.\nSecond line.")
|
| 17 |
-
from src.readers.text import read_text
|
| 18 |
-
pages = read_text(str(f))
|
| 19 |
-
assert len(pages) >= 1
|
| 20 |
-
assert "test document" in pages[0]["text"]
|
| 21 |
-
assert pages[0]["page"] == 1
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
def test_text_reader_json(tmp_path):
|
| 25 |
-
f = tmp_path / "data.json"
|
| 26 |
-
f.write_text('{"key": "value", "items": [1, 2, 3]}')
|
| 27 |
-
from src.readers.text import read_text
|
| 28 |
-
pages = read_text(str(f))
|
| 29 |
-
assert len(pages) >= 1
|
| 30 |
-
assert "key" in pages[0]["text"]
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def test_text_reader_empty(tmp_path):
|
| 34 |
-
f = tmp_path / "empty.txt"
|
| 35 |
-
f.write_text("")
|
| 36 |
-
from src.readers.text import read_text
|
| 37 |
-
pages = read_text(str(f))
|
| 38 |
-
assert pages == []
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
def test_reader_registry_dispatch(tmp_path):
|
| 42 |
-
f = tmp_path / "test.txt"
|
| 43 |
-
f.write_text("Hello world")
|
| 44 |
-
from src.readers import read_file
|
| 45 |
-
pages = read_file(str(f))
|
| 46 |
-
assert len(pages) >= 1
|
| 47 |
-
assert "Hello world" in pages[0]["text"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_readers_docx_excel_csv.py
DELETED
|
@@ -1,47 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_docx_reader(tmp_path):
|
| 4 |
-
pytest.importorskip("docx")
|
| 5 |
-
from docx import Document
|
| 6 |
-
doc = Document()
|
| 7 |
-
doc.add_paragraph("First paragraph of the document.")
|
| 8 |
-
doc.add_paragraph("Second paragraph with more content.")
|
| 9 |
-
path = tmp_path / "test.docx"
|
| 10 |
-
doc.save(str(path))
|
| 11 |
-
from src.readers.docx import read_docx
|
| 12 |
-
pages = read_docx(str(path))
|
| 13 |
-
assert len(pages) >= 1
|
| 14 |
-
assert "First paragraph" in pages[0]["text"]
|
| 15 |
-
|
| 16 |
-
def test_xlsx_reader(tmp_path):
|
| 17 |
-
pytest.importorskip("openpyxl")
|
| 18 |
-
import openpyxl
|
| 19 |
-
wb = openpyxl.Workbook()
|
| 20 |
-
ws = wb.active
|
| 21 |
-
ws.title = "Data"
|
| 22 |
-
ws.append(["Country", "Year", "Success"])
|
| 23 |
-
ws.append(["India", 1930, "Yes"])
|
| 24 |
-
ws.append(["Poland", 1980, "Yes"])
|
| 25 |
-
path = tmp_path / "test.xlsx"
|
| 26 |
-
wb.save(str(path))
|
| 27 |
-
from src.readers.excel import read_excel
|
| 28 |
-
pages = read_excel(str(path))
|
| 29 |
-
assert len(pages) >= 1
|
| 30 |
-
assert "India" in pages[0]["text"]
|
| 31 |
-
assert "Country" in pages[0]["text"]
|
| 32 |
-
|
| 33 |
-
def test_csv_reader(tmp_path):
|
| 34 |
-
f = tmp_path / "data.csv"
|
| 35 |
-
f.write_text("name,year,outcome\nAlice,2020,success\nBob,2021,failure\n")
|
| 36 |
-
from src.readers.csv_tab import read_csv_tab
|
| 37 |
-
pages = read_csv_tab(str(f))
|
| 38 |
-
assert len(pages) >= 1
|
| 39 |
-
assert "Alice" in pages[0]["text"]
|
| 40 |
-
|
| 41 |
-
def test_tab_reader(tmp_path):
|
| 42 |
-
f = tmp_path / "data.tab"
|
| 43 |
-
f.write_text("country\tyear\tresult\nEgypt\t2011\tpartial\n")
|
| 44 |
-
from src.readers.csv_tab import read_csv_tab
|
| 45 |
-
pages = read_csv_tab(str(f), delimiter="\t")
|
| 46 |
-
assert len(pages) >= 1
|
| 47 |
-
assert "Egypt" in pages[0]["text"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_readers_stat.py
DELETED
|
@@ -1,38 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_stata_reader(tmp_path):
|
| 4 |
-
pytest.importorskip("pyreadstat")
|
| 5 |
-
import pandas as pd
|
| 6 |
-
import pyreadstat
|
| 7 |
-
df = pd.DataFrame({"country": ["India", "Poland"], "year": [1930, 1980]})
|
| 8 |
-
path = str(tmp_path / "test.dta")
|
| 9 |
-
pyreadstat.write_dta(df, path)
|
| 10 |
-
from src.readers.stata import read_stata
|
| 11 |
-
pages = read_stata(path)
|
| 12 |
-
assert len(pages) >= 1
|
| 13 |
-
assert "India" in pages[0]["text"] or "India" in pages[1]["text"]
|
| 14 |
-
|
| 15 |
-
def test_spss_reader(tmp_path):
|
| 16 |
-
pytest.importorskip("pyreadstat")
|
| 17 |
-
import pandas as pd
|
| 18 |
-
import pyreadstat
|
| 19 |
-
df = pd.DataFrame({"name": ["Alice", "Bob"], "score": [85.0, 92.0]})
|
| 20 |
-
path = str(tmp_path / "test.sav")
|
| 21 |
-
pyreadstat.write_sav(df, path)
|
| 22 |
-
from src.readers.spss import read_spss
|
| 23 |
-
pages = read_spss(path)
|
| 24 |
-
assert len(pages) >= 1
|
| 25 |
-
found = any("Alice" in p["text"] for p in pages)
|
| 26 |
-
assert found
|
| 27 |
-
|
| 28 |
-
def test_rdata_reader(tmp_path):
|
| 29 |
-
pytest.importorskip("pyreadr")
|
| 30 |
-
import pandas as pd
|
| 31 |
-
import pyreadr
|
| 32 |
-
df = pd.DataFrame({"x": [1, 2, 3], "y": ["a", "b", "c"]})
|
| 33 |
-
path = str(tmp_path / "test.rds")
|
| 34 |
-
pyreadr.write_rds(path, df)
|
| 35 |
-
from src.readers.rdata import read_rdata
|
| 36 |
-
pages = read_rdata(path)
|
| 37 |
-
assert len(pages) >= 1
|
| 38 |
-
assert "x" in pages[0]["text"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_retriever.py
DELETED
|
@@ -1,59 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_format_db_results_as_context_empty():
|
| 4 |
-
from src.retriever import format_db_results_as_context
|
| 5 |
-
assert format_db_results_as_context([]) == ""
|
| 6 |
-
|
| 7 |
-
def test_format_db_results_as_context_with_chunks():
|
| 8 |
-
from src.retriever import format_db_results_as_context
|
| 9 |
-
chunks = [{
|
| 10 |
-
"text": "Test content here.",
|
| 11 |
-
"metadata": {"source": "Dataset1/paper.pdf", "dataset": "Dataset1", "page": "3"},
|
| 12 |
-
"distance": 0.1,
|
| 13 |
-
}]
|
| 14 |
-
context = format_db_results_as_context(chunks)
|
| 15 |
-
assert "[CHUNK-LOCAL-001]" in context
|
| 16 |
-
assert "paper.pdf" in context
|
| 17 |
-
assert "Test content here." in context
|
| 18 |
-
assert "PRIMARY" in context
|
| 19 |
-
|
| 20 |
-
def test_build_combined_context_no_sources():
|
| 21 |
-
from src.retriever import build_combined_context
|
| 22 |
-
result = build_combined_context([], [])
|
| 23 |
-
assert "No relevant" in result or "don't have" in result
|
| 24 |
-
|
| 25 |
-
def test_build_combined_context_local_only():
|
| 26 |
-
from src.retriever import build_combined_context
|
| 27 |
-
db = [{"text": "data", "metadata": {"source": "f.pdf", "dataset": "d", "page": "1"}, "distance": 0.1}]
|
| 28 |
-
result = build_combined_context(db, [])
|
| 29 |
-
assert "ONLY" in result or "local" in result.lower()
|
| 30 |
-
|
| 31 |
-
def test_build_combined_context_both():
|
| 32 |
-
from src.retriever import build_combined_context
|
| 33 |
-
db = [{"text": "data", "metadata": {"source": "f.pdf", "dataset": "d", "page": "1"}, "distance": 0.1}]
|
| 34 |
-
web = [{"title": "P", "authors": "A", "year": 2020, "abstract": "abs", "url": "http://x", "citation_count": 1, "source_type": "web_search"}]
|
| 35 |
-
result = build_combined_context(db, web)
|
| 36 |
-
assert "PRIMARY" in result
|
| 37 |
-
assert "SUPPLEMENTARY" in result
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def test_build_combined_context_with_sql():
|
| 41 |
-
from src.retriever import build_combined_context
|
| 42 |
-
sql = "=== SQL Query Results (PRIMARY) ===\n[CHUNK-SQL-001] x = 1"
|
| 43 |
-
db = [{"text": "data", "metadata": {"source": "f.pdf", "dataset": "d", "page": "1"}, "distance": 0.1}]
|
| 44 |
-
result = build_combined_context(db, [], sql_context=sql)
|
| 45 |
-
assert "SQL Query Results" in result
|
| 46 |
-
assert "CHUNK-LOCAL" in result
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
def test_build_combined_context_sql_only():
|
| 50 |
-
from src.retriever import build_combined_context
|
| 51 |
-
sql = "=== SQL Query Results (PRIMARY) ===\n[CHUNK-SQL-001] x = 1"
|
| 52 |
-
result = build_combined_context([], [], sql_context=sql)
|
| 53 |
-
assert "SQL Query Results" in result
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
def test_build_combined_context_sql_no_sources():
|
| 57 |
-
from src.retriever import build_combined_context
|
| 58 |
-
result = build_combined_context([], [], sql_context="")
|
| 59 |
-
assert "No relevant" in result or "don't have" in result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_search.py
DELETED
|
@@ -1,28 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_search_registry_has_backends():
|
| 4 |
-
from src.search import BACKENDS
|
| 5 |
-
assert "semantic_scholar" in BACKENDS
|
| 6 |
-
assert "none" in BACKENDS
|
| 7 |
-
|
| 8 |
-
def test_search_none_returns_empty():
|
| 9 |
-
from src.search import search
|
| 10 |
-
results = search("test query", backend="none")
|
| 11 |
-
assert results == []
|
| 12 |
-
|
| 13 |
-
def test_format_web_results_empty():
|
| 14 |
-
from src.search import format_web_results_as_context
|
| 15 |
-
assert format_web_results_as_context([]) == ""
|
| 16 |
-
|
| 17 |
-
def test_format_web_results_with_data():
|
| 18 |
-
from src.search import format_web_results_as_context
|
| 19 |
-
results = [{
|
| 20 |
-
"title": "Test Paper", "authors": "Smith, J.", "year": 2023,
|
| 21 |
-
"abstract": "An abstract.", "url": "https://doi.org/10.1234/test",
|
| 22 |
-
"citation_count": 10, "source_type": "web_search",
|
| 23 |
-
}]
|
| 24 |
-
context = format_web_results_as_context(results)
|
| 25 |
-
assert "Test Paper" in context
|
| 26 |
-
assert "Smith" in context
|
| 27 |
-
assert "https://doi.org/10.1234/test" in context
|
| 28 |
-
assert "[CHUNK-WEB-" in context
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_setup.py
DELETED
|
@@ -1,39 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
|
| 3 |
-
def test_generate_config_yaml():
|
| 4 |
-
from setup import generate_config
|
| 5 |
-
import yaml
|
| 6 |
-
config_str = generate_config(
|
| 7 |
-
bot_name="TestBot", domain="testing",
|
| 8 |
-
provider="openai", model="gpt-4o", web_search=True,
|
| 9 |
-
)
|
| 10 |
-
cfg = yaml.safe_load(config_str)
|
| 11 |
-
assert cfg["chatbot"]["name"] == "TestBot"
|
| 12 |
-
assert cfg["llm"]["provider"] == "openai"
|
| 13 |
-
assert cfg["llm"]["model"] == "gpt-4o"
|
| 14 |
-
assert cfg["web_search"]["enabled"] is True
|
| 15 |
-
assert cfg["verification"]["enabled"] is True
|
| 16 |
-
assert cfg["paths"]["knowledge_base"] == "knowledge_base"
|
| 17 |
-
# Ensure field names match what the rest of the codebase reads
|
| 18 |
-
assert cfg["paths"]["vector_db"] == "chroma_db"
|
| 19 |
-
assert cfg["web_search"]["backend"] == "semantic_scholar"
|
| 20 |
-
assert cfg["verification"]["max_iterations"] == 3
|
| 21 |
-
assert cfg["verification"]["strict_mode"] is True
|
| 22 |
-
assert cfg["retrieval"]["chunk_size"] == 1000
|
| 23 |
-
assert cfg["retrieval"]["chunk_overlap"] == 100
|
| 24 |
-
assert cfg["retrieval"]["top_k"] == 50
|
| 25 |
-
assert cfg["retrieval"]["max_distance"] == 0.55
|
| 26 |
-
assert cfg["embeddings"]["provider"] == "local"
|
| 27 |
-
# Query understanding config
|
| 28 |
-
assert cfg["query_understanding"]["enabled"] is True
|
| 29 |
-
assert cfg["query_understanding"]["max_history"] == 6
|
| 30 |
-
assert cfg["query_understanding"]["max_clarifications"] == 1
|
| 31 |
-
# SQL config
|
| 32 |
-
assert cfg["paths"]["sql_db"] == "sql_db"
|
| 33 |
-
assert cfg["sql"]["enabled"] is True
|
| 34 |
-
assert cfg["sql"]["max_rows"] == 200
|
| 35 |
-
|
| 36 |
-
def test_generate_env_file():
|
| 37 |
-
from setup import generate_env
|
| 38 |
-
env_str = generate_env("openai", "sk-test-123")
|
| 39 |
-
assert "OPENAI_API_KEY=sk-test-123" in env_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_sql_ingest.py
DELETED
|
@@ -1,204 +0,0 @@
|
|
| 1 |
-
# tests/test_sql_ingest.py
|
| 2 |
-
import pytest
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
def test_sanitize_table_name_basic():
|
| 6 |
-
from src.sql_ingest import _sanitize_table_name
|
| 7 |
-
assert _sanitize_table_name("PTS_dataset", "pts_data", ".csv") == "pts_dataset__pts_data_csv"
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def test_sanitize_table_name_with_sheet():
|
| 11 |
-
from src.sql_ingest import _sanitize_table_name
|
| 12 |
-
result = _sanitize_table_name("econ", "gdp", ".xlsx", "Sheet1")
|
| 13 |
-
assert result == "econ__gdp_xlsx_sheet1"
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def test_sanitize_table_name_special_chars():
|
| 17 |
-
from src.sql_ingest import _sanitize_table_name
|
| 18 |
-
result = _sanitize_table_name("my-data", "file (2)", ".csv")
|
| 19 |
-
assert result == "my_data__file_2_csv"
|
| 20 |
-
assert result[0].isalpha()
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
def test_sanitize_table_name_starts_with_number():
|
| 24 |
-
from src.sql_ingest import _sanitize_table_name
|
| 25 |
-
result = _sanitize_table_name("123data", "file", ".csv")
|
| 26 |
-
assert result[0].isalpha()
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
def test_infer_column_type_integer():
|
| 30 |
-
from src.sql_ingest import _infer_column_type
|
| 31 |
-
assert _infer_column_type(["1", "2", "3", None, ""]) == "INTEGER"
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
def test_infer_column_type_real():
|
| 35 |
-
from src.sql_ingest import _infer_column_type
|
| 36 |
-
assert _infer_column_type(["1.5", "2.0", "3.7"]) == "REAL"
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
def test_infer_column_type_text():
|
| 40 |
-
from src.sql_ingest import _infer_column_type
|
| 41 |
-
assert _infer_column_type(["China", "India", "Brazil"]) == "TEXT"
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
def test_infer_column_type_mixed_defaults_text():
|
| 45 |
-
from src.sql_ingest import _infer_column_type
|
| 46 |
-
assert _infer_column_type(["1", "two", "3"]) == "TEXT"
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
def test_infer_column_type_empty():
|
| 50 |
-
from src.sql_ingest import _infer_column_type
|
| 51 |
-
assert _infer_column_type([None, "", "nan"]) == "TEXT"
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def test_get_sample_values():
|
| 55 |
-
from src.sql_ingest import _get_sample_values
|
| 56 |
-
samples = _get_sample_values(["China", "India", "China", "Brazil", None, ""], n=3)
|
| 57 |
-
assert samples == ["China", "India", "Brazil"]
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def test_get_sample_values_fewer_than_n():
|
| 61 |
-
from src.sql_ingest import _get_sample_values
|
| 62 |
-
samples = _get_sample_values(["a", None, "a"], n=3)
|
| 63 |
-
assert samples == ["a"]
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
import csv
|
| 67 |
-
import json
|
| 68 |
-
import sqlite3
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
def test_ingest_to_sql_creates_db_and_schema(tmp_path):
|
| 72 |
-
from pathlib import Path
|
| 73 |
-
from src.sql_ingest import ingest_to_sql
|
| 74 |
-
|
| 75 |
-
# Create a CSV file
|
| 76 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 77 |
-
ds_dir = kb_dir / "testds"
|
| 78 |
-
ds_dir.mkdir(parents=True)
|
| 79 |
-
csv_file = ds_dir / "data.csv"
|
| 80 |
-
csv_file.write_text("Country,Year,Score\nChina,2005,4.0\nIndia,2005,3.0\n")
|
| 81 |
-
|
| 82 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 83 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 84 |
-
files = [(Path(csv_file), "testds")]
|
| 85 |
-
|
| 86 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 87 |
-
|
| 88 |
-
# Check DB was created
|
| 89 |
-
db_path = sql_db_dir / "knowledge_base.db"
|
| 90 |
-
assert db_path.exists()
|
| 91 |
-
|
| 92 |
-
# Check schema registry
|
| 93 |
-
schema_path = sql_db_dir / "sql_schemas.json"
|
| 94 |
-
assert schema_path.exists()
|
| 95 |
-
with open(schema_path) as f:
|
| 96 |
-
saved = json.load(f)
|
| 97 |
-
assert len(saved) == 1
|
| 98 |
-
table_name = list(saved.keys())[0]
|
| 99 |
-
assert "testds" in table_name
|
| 100 |
-
assert saved[table_name]["row_count"] == 2
|
| 101 |
-
assert len(saved[table_name]["columns"]) == 3
|
| 102 |
-
|
| 103 |
-
# Check data in SQLite
|
| 104 |
-
conn = sqlite3.connect(str(db_path))
|
| 105 |
-
rows = conn.execute(f'SELECT * FROM "{table_name}"').fetchall()
|
| 106 |
-
conn.close()
|
| 107 |
-
assert len(rows) == 2
|
| 108 |
-
assert rows[0][0] == "China"
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
def test_ingest_to_sql_type_inference(tmp_path):
|
| 112 |
-
from pathlib import Path
|
| 113 |
-
from src.sql_ingest import ingest_to_sql
|
| 114 |
-
|
| 115 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 116 |
-
ds_dir = kb_dir / "ds"
|
| 117 |
-
ds_dir.mkdir(parents=True)
|
| 118 |
-
csv_file = ds_dir / "typed.csv"
|
| 119 |
-
csv_file.write_text("Name,Year,Score\nChina,2005,4.5\nIndia,2006,3.0\n")
|
| 120 |
-
|
| 121 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 122 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 123 |
-
files = [(Path(csv_file), "ds")]
|
| 124 |
-
|
| 125 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 126 |
-
table_name = list(schema.keys())[0]
|
| 127 |
-
|
| 128 |
-
# Name=TEXT, Year=INTEGER, Score=REAL
|
| 129 |
-
col_types = {c["name"]: c["type"] for c in schema[table_name]["columns"]}
|
| 130 |
-
assert col_types["Name"] == "TEXT"
|
| 131 |
-
assert col_types["Year"] == "INTEGER"
|
| 132 |
-
assert col_types["Score"] == "REAL"
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
def test_ingest_to_sql_no_tabular_files(tmp_path):
|
| 136 |
-
from pathlib import Path
|
| 137 |
-
from src.sql_ingest import ingest_to_sql
|
| 138 |
-
|
| 139 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 140 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 141 |
-
# Pass empty list (no tabular files)
|
| 142 |
-
schema = ingest_to_sql([], str(tmp_path), cfg)
|
| 143 |
-
assert schema == {}
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
def test_ingest_to_sql_clears_on_rerun(tmp_path):
|
| 147 |
-
from pathlib import Path
|
| 148 |
-
from src.sql_ingest import ingest_to_sql
|
| 149 |
-
|
| 150 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 151 |
-
ds_dir = kb_dir / "ds"
|
| 152 |
-
ds_dir.mkdir(parents=True)
|
| 153 |
-
csv_file = ds_dir / "data.csv"
|
| 154 |
-
csv_file.write_text("A,B\n1,2\n3,4\n")
|
| 155 |
-
|
| 156 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 157 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}}
|
| 158 |
-
files = [(Path(csv_file), "ds")]
|
| 159 |
-
|
| 160 |
-
# First run
|
| 161 |
-
ingest_to_sql(files, str(kb_dir), cfg)
|
| 162 |
-
# Second run — should not duplicate
|
| 163 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 164 |
-
table_name = list(schema.keys())[0]
|
| 165 |
-
assert schema[table_name]["row_count"] == 2
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
def test_ingest_documents_calls_sql_ingest(tmp_path):
|
| 169 |
-
"""Integration test: ingest_documents should also populate SQLite."""
|
| 170 |
-
import os
|
| 171 |
-
from pathlib import Path
|
| 172 |
-
|
| 173 |
-
# Set up knowledge base with a CSV
|
| 174 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 175 |
-
ds_dir = kb_dir / "testds"
|
| 176 |
-
ds_dir.mkdir(parents=True)
|
| 177 |
-
csv_file = ds_dir / "scores.csv"
|
| 178 |
-
csv_file.write_text("Country,Year,Score\nChina,2005,4.0\nIndia,2005,3.0\n")
|
| 179 |
-
|
| 180 |
-
# Minimal config
|
| 181 |
-
cfg = {
|
| 182 |
-
"paths": {
|
| 183 |
-
"knowledge_base": str(kb_dir),
|
| 184 |
-
"vector_db": str(tmp_path / "chroma_db"),
|
| 185 |
-
"sql_db": str(tmp_path / "sql_db"),
|
| 186 |
-
},
|
| 187 |
-
"retrieval": {"chunk_size": 1000, "chunk_overlap": 100},
|
| 188 |
-
"embeddings": {"provider": "local"},
|
| 189 |
-
"sql": {"enabled": True},
|
| 190 |
-
}
|
| 191 |
-
|
| 192 |
-
from src.ingest import ingest_documents
|
| 193 |
-
count = ingest_documents(cfg, documents_dir=str(kb_dir))
|
| 194 |
-
|
| 195 |
-
# Verify ChromaDB got chunks
|
| 196 |
-
assert count > 0
|
| 197 |
-
|
| 198 |
-
# Verify SQLite DB was created
|
| 199 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 200 |
-
assert db_path.exists()
|
| 201 |
-
|
| 202 |
-
# Verify schema registry
|
| 203 |
-
schema_path = tmp_path / "sql_db" / "sql_schemas.json"
|
| 204 |
-
assert schema_path.exists()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_sql_integration.py
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
# tests/test_sql_integration.py
|
| 2 |
-
"""End-to-end integration test for the SQL layer."""
|
| 3 |
-
import json
|
| 4 |
-
import sqlite3
|
| 5 |
-
import pytest
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
def test_sql_roundtrip(tmp_path):
|
| 9 |
-
"""Test: ingest CSV -> SQL -> retrieve -> format context."""
|
| 10 |
-
from pathlib import Path
|
| 11 |
-
from src.sql_ingest import ingest_to_sql
|
| 12 |
-
from src.sql_retriever import execute_sql_query, format_sql_results_as_context, build_schema_summary
|
| 13 |
-
|
| 14 |
-
# Create test data
|
| 15 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 16 |
-
ds_dir = kb_dir / "pts"
|
| 17 |
-
ds_dir.mkdir(parents=True)
|
| 18 |
-
csv_file = ds_dir / "scores.csv"
|
| 19 |
-
csv_file.write_text(
|
| 20 |
-
"Country,Year,PTS_A\n"
|
| 21 |
-
"China,2000,4.0\n"
|
| 22 |
-
"China,2001,4.0\n"
|
| 23 |
-
"China,2002,3.5\n"
|
| 24 |
-
"India,2000,3.0\n"
|
| 25 |
-
"India,2001,3.0\n"
|
| 26 |
-
)
|
| 27 |
-
|
| 28 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 29 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}, "sql": {"max_rows": 200}}
|
| 30 |
-
files = [(Path(csv_file), "pts")]
|
| 31 |
-
|
| 32 |
-
# Ingest
|
| 33 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 34 |
-
assert len(schema) == 1
|
| 35 |
-
table_name = list(schema.keys())[0]
|
| 36 |
-
|
| 37 |
-
# Build schema summary
|
| 38 |
-
summary = build_schema_summary(schema)
|
| 39 |
-
assert "Country" in summary
|
| 40 |
-
assert "PTS_A" in summary
|
| 41 |
-
|
| 42 |
-
# Execute a query
|
| 43 |
-
sql = f'SELECT Country, Year, PTS_A FROM "{table_name}" WHERE Country = \'China\' ORDER BY Year'
|
| 44 |
-
rows = execute_sql_query(sql, cfg)
|
| 45 |
-
assert len(rows) == 3
|
| 46 |
-
assert all(r["Country"] == "China" for r in rows)
|
| 47 |
-
|
| 48 |
-
# Format as context
|
| 49 |
-
context = format_sql_results_as_context(rows, sql, schema[table_name]["source_file"])
|
| 50 |
-
assert "[CHUNK-SQL-001]" in context
|
| 51 |
-
assert "China" in context
|
| 52 |
-
assert "pts/scores.csv" in context
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def test_sql_aggregation(tmp_path):
|
| 56 |
-
"""Test: aggregation queries work."""
|
| 57 |
-
from pathlib import Path
|
| 58 |
-
from src.sql_ingest import ingest_to_sql
|
| 59 |
-
from src.sql_retriever import execute_sql_query
|
| 60 |
-
|
| 61 |
-
kb_dir = tmp_path / "knowledge_base"
|
| 62 |
-
ds_dir = kb_dir / "ds"
|
| 63 |
-
ds_dir.mkdir(parents=True)
|
| 64 |
-
csv_file = ds_dir / "data.csv"
|
| 65 |
-
csv_file.write_text("Country,Year,Score\nA,2000,1\nB,2000,2\nC,2000,3\n")
|
| 66 |
-
|
| 67 |
-
sql_db_dir = tmp_path / "sql_db"
|
| 68 |
-
cfg = {"paths": {"sql_db": str(sql_db_dir)}, "sql": {"max_rows": 200}}
|
| 69 |
-
files = [(Path(csv_file), "ds")]
|
| 70 |
-
schema = ingest_to_sql(files, str(kb_dir), cfg)
|
| 71 |
-
table_name = list(schema.keys())[0]
|
| 72 |
-
|
| 73 |
-
# Count
|
| 74 |
-
rows = execute_sql_query(f'SELECT COUNT(*) as cnt FROM "{table_name}"', cfg)
|
| 75 |
-
assert rows[0]["cnt"] == 3
|
| 76 |
-
|
| 77 |
-
# Average
|
| 78 |
-
rows = execute_sql_query(f'SELECT AVG(Score) as avg_score FROM "{table_name}"', cfg)
|
| 79 |
-
assert rows[0]["avg_score"] == 2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_sql_retriever.py
DELETED
|
@@ -1,159 +0,0 @@
|
|
| 1 |
-
# tests/test_sql_retriever.py
|
| 2 |
-
import pytest
|
| 3 |
-
import json
|
| 4 |
-
import sqlite3
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
def test_validate_sql_rejects_non_select():
|
| 8 |
-
from src.sql_retriever import _validate_sql
|
| 9 |
-
assert _validate_sql("DROP TABLE users") is False
|
| 10 |
-
assert _validate_sql("INSERT INTO t VALUES (1)") is False
|
| 11 |
-
assert _validate_sql("UPDATE t SET x=1") is False
|
| 12 |
-
assert _validate_sql("DELETE FROM t") is False
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def test_validate_sql_rejects_semicolons():
|
| 16 |
-
from src.sql_retriever import _validate_sql
|
| 17 |
-
assert _validate_sql("SELECT 1; DROP TABLE users") is False
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def test_validate_sql_accepts_select():
|
| 21 |
-
from src.sql_retriever import _validate_sql
|
| 22 |
-
assert _validate_sql("SELECT * FROM t") is True
|
| 23 |
-
assert _validate_sql(" select Country, Year from t WHERE x = 1 ") is True
|
| 24 |
-
assert _validate_sql("SELECT COUNT(*) FROM t GROUP BY x") is True
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
def test_validate_sql_rejects_empty():
|
| 28 |
-
from src.sql_retriever import _validate_sql
|
| 29 |
-
assert _validate_sql("") is False
|
| 30 |
-
assert _validate_sql(" ") is False
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def test_execute_sql_query_returns_rows(tmp_path):
|
| 34 |
-
from src.sql_retriever import execute_sql_query
|
| 35 |
-
|
| 36 |
-
# Create a test DB
|
| 37 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 38 |
-
db_path.parent.mkdir()
|
| 39 |
-
conn = sqlite3.connect(str(db_path))
|
| 40 |
-
conn.execute("CREATE TABLE t (Country TEXT, Year INTEGER, Score REAL)")
|
| 41 |
-
conn.execute("INSERT INTO t VALUES ('China', 2005, 4.0)")
|
| 42 |
-
conn.execute("INSERT INTO t VALUES ('India', 2005, 3.0)")
|
| 43 |
-
conn.commit()
|
| 44 |
-
conn.close()
|
| 45 |
-
|
| 46 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}, "sql": {"max_rows": 200}}
|
| 47 |
-
rows = execute_sql_query("SELECT * FROM t WHERE Country = 'China'", cfg)
|
| 48 |
-
assert len(rows) == 1
|
| 49 |
-
assert rows[0]["Country"] == "China"
|
| 50 |
-
assert rows[0]["Score"] == 4.0
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def test_execute_sql_query_respects_max_rows(tmp_path):
|
| 54 |
-
from src.sql_retriever import execute_sql_query
|
| 55 |
-
|
| 56 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 57 |
-
db_path.parent.mkdir()
|
| 58 |
-
conn = sqlite3.connect(str(db_path))
|
| 59 |
-
conn.execute("CREATE TABLE t (x INTEGER)")
|
| 60 |
-
for i in range(100):
|
| 61 |
-
conn.execute("INSERT INTO t VALUES (?)", (i,))
|
| 62 |
-
conn.commit()
|
| 63 |
-
conn.close()
|
| 64 |
-
|
| 65 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}, "sql": {"max_rows": 5}}
|
| 66 |
-
rows = execute_sql_query("SELECT * FROM t", cfg)
|
| 67 |
-
assert len(rows) == 5
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
def test_execute_sql_query_rejects_non_select(tmp_path):
|
| 71 |
-
from src.sql_retriever import execute_sql_query
|
| 72 |
-
|
| 73 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 74 |
-
db_path.parent.mkdir()
|
| 75 |
-
conn = sqlite3.connect(str(db_path))
|
| 76 |
-
conn.execute("CREATE TABLE t (x INTEGER)")
|
| 77 |
-
conn.commit()
|
| 78 |
-
conn.close()
|
| 79 |
-
|
| 80 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}}
|
| 81 |
-
rows = execute_sql_query("DROP TABLE t", cfg)
|
| 82 |
-
assert rows == []
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
def test_execute_sql_query_handles_bad_sql(tmp_path):
|
| 86 |
-
from src.sql_retriever import execute_sql_query
|
| 87 |
-
|
| 88 |
-
db_path = tmp_path / "sql_db" / "knowledge_base.db"
|
| 89 |
-
db_path.parent.mkdir()
|
| 90 |
-
conn = sqlite3.connect(str(db_path))
|
| 91 |
-
conn.execute("CREATE TABLE t (x INTEGER)")
|
| 92 |
-
conn.commit()
|
| 93 |
-
conn.close()
|
| 94 |
-
|
| 95 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db")}}
|
| 96 |
-
rows = execute_sql_query("SELECT * FROM nonexistent_table", cfg)
|
| 97 |
-
assert rows == []
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
def test_execute_sql_query_no_db(tmp_path):
|
| 101 |
-
from src.sql_retriever import execute_sql_query
|
| 102 |
-
|
| 103 |
-
cfg = {"paths": {"sql_db": str(tmp_path / "sql_db_missing")}}
|
| 104 |
-
rows = execute_sql_query("SELECT 1", cfg)
|
| 105 |
-
assert rows == []
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
def test_format_sql_results_empty():
|
| 109 |
-
from src.sql_retriever import format_sql_results_as_context
|
| 110 |
-
assert format_sql_results_as_context([], "SELECT 1", "data.csv") == ""
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
def test_format_sql_results_basic():
|
| 114 |
-
from src.sql_retriever import format_sql_results_as_context
|
| 115 |
-
rows = [
|
| 116 |
-
{"Country": "China", "Year": 2005, "PTS_A": 4.0},
|
| 117 |
-
{"Country": "China", "Year": 2006, "PTS_A": 3.5},
|
| 118 |
-
]
|
| 119 |
-
result = format_sql_results_as_context(rows, "SELECT * FROM t", "PTS_dataset/pts_data.csv")
|
| 120 |
-
assert "[CHUNK-SQL-001]" in result
|
| 121 |
-
assert "[CHUNK-SQL-002]" in result
|
| 122 |
-
assert "China" in result
|
| 123 |
-
assert "PTS_dataset/pts_data.csv" in result
|
| 124 |
-
assert "PRIMARY" in result
|
| 125 |
-
assert "SELECT * FROM t" in result
|
| 126 |
-
assert "Rows returned: 2" in result
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
def test_build_schema_summary_empty():
|
| 130 |
-
from src.sql_retriever import build_schema_summary
|
| 131 |
-
assert build_schema_summary({}) == ""
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
def test_build_schema_summary_basic():
|
| 135 |
-
from src.sql_retriever import build_schema_summary
|
| 136 |
-
schema = {
|
| 137 |
-
"test_table": {
|
| 138 |
-
"source_file": "data.csv",
|
| 139 |
-
"columns": [
|
| 140 |
-
{"name": "Country", "type": "TEXT", "sample": ["China", "India"]},
|
| 141 |
-
{"name": "Year", "type": "INTEGER", "sample": [2005, 2010]},
|
| 142 |
-
],
|
| 143 |
-
"row_count": 100,
|
| 144 |
-
}
|
| 145 |
-
}
|
| 146 |
-
result = build_schema_summary(schema)
|
| 147 |
-
assert "test_table" in result
|
| 148 |
-
assert "100 rows" in result
|
| 149 |
-
assert "Country (TEXT)" in result
|
| 150 |
-
assert "Year (INTEGER)" in result
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
def test_lookup_source_file():
|
| 154 |
-
from src.sql_retriever import _lookup_source_file
|
| 155 |
-
schema = {
|
| 156 |
-
"ds__data_csv": {"source_file": "ds/data.csv", "columns": [], "row_count": 0},
|
| 157 |
-
}
|
| 158 |
-
assert _lookup_source_file("ds__data_csv", schema) == "ds/data.csv"
|
| 159 |
-
assert _lookup_source_file("unknown", schema) == "unknown"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_verifier.py
DELETED
|
@@ -1,54 +0,0 @@
|
|
| 1 |
-
import pytest
|
| 2 |
-
import json
|
| 3 |
-
|
| 4 |
-
def test_scan_warning_phrases_clean():
|
| 5 |
-
from src.verifier import scan_warning_phrases
|
| 6 |
-
flags = scan_warning_phrases("The data shows a 53% success rate. [1]")
|
| 7 |
-
assert flags == []
|
| 8 |
-
|
| 9 |
-
def test_scan_warning_phrases_flagged():
|
| 10 |
-
from src.verifier import scan_warning_phrases
|
| 11 |
-
flags = scan_warning_phrases("It is well known that nonviolent movements work.")
|
| 12 |
-
assert len(flags) == 1
|
| 13 |
-
assert flags[0]["severity"] == "advisory"
|
| 14 |
-
|
| 15 |
-
def test_compute_soft_max_tokens():
|
| 16 |
-
from src.verifier import compute_soft_max_tokens
|
| 17 |
-
assert compute_soft_max_tokens(100, 2048) == 1024
|
| 18 |
-
assert compute_soft_max_tokens(1500, 2048) == 1536
|
| 19 |
-
assert compute_soft_max_tokens(5000, 2048) == 2048
|
| 20 |
-
|
| 21 |
-
def test_parse_verification_result_pass():
|
| 22 |
-
from src.verifier import parse_verification_result
|
| 23 |
-
json_str = '{"errors": [], "error_count": 0, "pass": true}'
|
| 24 |
-
result = parse_verification_result(json_str)
|
| 25 |
-
assert result["pass"] is True
|
| 26 |
-
assert result["error_count"] == 0
|
| 27 |
-
|
| 28 |
-
def test_parse_verification_result_fail():
|
| 29 |
-
from src.verifier import parse_verification_result
|
| 30 |
-
json_str = json.dumps({
|
| 31 |
-
"errors": [{"claim": "test", "issue": "not found in sources"}],
|
| 32 |
-
"error_count": 1, "pass": False,
|
| 33 |
-
})
|
| 34 |
-
result = parse_verification_result(json_str)
|
| 35 |
-
assert result["pass"] is False
|
| 36 |
-
assert result["error_count"] == 1
|
| 37 |
-
|
| 38 |
-
def test_parse_verification_result_malformed():
|
| 39 |
-
from src.verifier import parse_verification_result
|
| 40 |
-
result = parse_verification_result("not json at all")
|
| 41 |
-
assert result["pass"] is False
|
| 42 |
-
|
| 43 |
-
def test_no_sources_returns_refusal():
|
| 44 |
-
from src.verifier import verify_and_respond
|
| 45 |
-
result = verify_and_respond(
|
| 46 |
-
query="test",
|
| 47 |
-
retrieval_result={"context": "", "db_results": [], "web_results": [], "has_sources": False},
|
| 48 |
-
cfg={"llm": {"provider": "openai", "model": "gpt-4o", "temperature": 0.0, "max_tokens": 2048},
|
| 49 |
-
"api_keys": {"openai": "fake"},
|
| 50 |
-
"verification": {"enabled": True, "max_iterations": 3, "strict_mode": True},
|
| 51 |
-
"chatbot": {"name": "Test", "domain": "test"}},
|
| 52 |
-
)
|
| 53 |
-
assert result["refused"] is True
|
| 54 |
-
assert "don't have any information" in result["response"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|