LIANJie-Jason Claude Opus 4.6 commited on
Commit
c65382c
·
1 Parent(s): 8529300

feat: add SQL schema loading and route/sql_query parsing to query engine

Browse files
Files changed (2) hide show
  1. src/query_engine.py +43 -1
  2. tests/test_query_engine.py +40 -0
src/query_engine.py CHANGED
@@ -1,12 +1,27 @@
1
  """Query understanding layer: reformulates user queries for better retrieval."""
2
 
3
  import json
 
4
  import re
5
 
6
  from src.llm import generate
7
  from src.prompts import build_query_understanding_prompt
8
 
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  def understand_query(
11
  user_query: str,
12
  cfg: dict,
@@ -40,6 +55,8 @@ def understand_query(
40
  "search_query": user_query,
41
  "display_query": user_query,
42
  "original_query": user_query,
 
 
43
  }
44
 
45
  domain = cfg.get("chatbot", {}).get("domain", "research")
@@ -50,7 +67,20 @@ def understand_query(
50
  if len(history) > max_history:
51
  history = history[-max_history:]
52
 
53
- prompt = build_query_understanding_prompt(user_query, domain, history)
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
  try:
56
  raw = generate(
@@ -67,6 +97,8 @@ def understand_query(
67
  "search_query": user_query,
68
  "display_query": user_query,
69
  "original_query": user_query,
 
 
70
  }
71
 
72
  return result
@@ -101,6 +133,8 @@ def _parse_qu_result(raw: str, original_query: str) -> dict:
101
  "search_query": original_query,
102
  "display_query": original_query,
103
  "original_query": original_query,
 
 
104
  }
105
 
106
  action = parsed.get("action", "search")
@@ -114,14 +148,22 @@ def _parse_qu_result(raw: str, original_query: str) -> dict:
114
  "clarification_question": parsed.get(
115
  "clarification_question", "Could you be more specific?"
116
  ),
 
 
117
  }
118
 
119
  # action == "search"
120
  search_query = parsed.get("search_query", original_query)
121
  display_query = parsed.get("display_query", original_query)
 
 
 
 
122
  return {
123
  "action": "search",
124
  "search_query": search_query,
125
  "display_query": display_query,
126
  "original_query": original_query,
 
 
127
  }
 
1
  """Query understanding layer: reformulates user queries for better retrieval."""
2
 
3
  import json
4
+ import os
5
  import re
6
 
7
  from src.llm import generate
8
  from src.prompts import build_query_understanding_prompt
9
 
10
 
11
+ def _load_sql_schema_summary(sql_db_dir: str) -> str:
12
+ """Load SQL schema summary from sql_schemas.json if it exists."""
13
+ schema_path = os.path.join(sql_db_dir, "sql_schemas.json")
14
+ if not os.path.exists(schema_path):
15
+ return ""
16
+ try:
17
+ with open(schema_path, "r", encoding="utf-8") as f:
18
+ schema = json.load(f)
19
+ from src.sql_retriever import build_schema_summary
20
+ return build_schema_summary(schema)
21
+ except Exception:
22
+ return ""
23
+
24
+
25
  def understand_query(
26
  user_query: str,
27
  cfg: dict,
 
55
  "search_query": user_query,
56
  "display_query": user_query,
57
  "original_query": user_query,
58
+ "route": "vector",
59
+ "sql_query": None,
60
  }
61
 
62
  domain = cfg.get("chatbot", {}).get("domain", "research")
 
67
  if len(history) > max_history:
68
  history = history[-max_history:]
69
 
70
+ # Load SQL schema summary for prompt injection
71
+ sql_enabled = cfg.get("sql", {}).get("enabled", True)
72
+ schema_summary = ""
73
+ if sql_enabled:
74
+ sql_db_dir = cfg.get("paths", {}).get("sql_db", "sql_db")
75
+ if not os.path.isabs(sql_db_dir):
76
+ from pathlib import Path as _Path
77
+ project_root = _Path(__file__).resolve().parent.parent
78
+ sql_db_dir = os.path.join(str(project_root), sql_db_dir)
79
+ schema_summary = _load_sql_schema_summary(sql_db_dir)
80
+
81
+ prompt = build_query_understanding_prompt(
82
+ user_query, domain, history, sql_schema_summary=schema_summary,
83
+ )
84
 
85
  try:
86
  raw = generate(
 
97
  "search_query": user_query,
98
  "display_query": user_query,
99
  "original_query": user_query,
100
+ "route": "vector",
101
+ "sql_query": None,
102
  }
103
 
104
  return result
 
133
  "search_query": original_query,
134
  "display_query": original_query,
135
  "original_query": original_query,
136
+ "route": "vector",
137
+ "sql_query": None,
138
  }
139
 
140
  action = parsed.get("action", "search")
 
148
  "clarification_question": parsed.get(
149
  "clarification_question", "Could you be more specific?"
150
  ),
151
+ "route": "vector",
152
+ "sql_query": None,
153
  }
154
 
155
  # action == "search"
156
  search_query = parsed.get("search_query", original_query)
157
  display_query = parsed.get("display_query", original_query)
158
+ route = parsed.get("route", "vector")
159
+ if route not in ("sql", "vector", "both"):
160
+ route = "vector"
161
+ sql_query = parsed.get("sql_query") if route in ("sql", "both") else None
162
  return {
163
  "action": "search",
164
  "search_query": search_query,
165
  "display_query": display_query,
166
  "original_query": original_query,
167
+ "route": route,
168
+ "sql_query": sql_query,
169
  }
tests/test_query_engine.py CHANGED
@@ -130,3 +130,43 @@ def test_parse_qu_result_with_display_query():
130
  assert result["search_query"] == "keyword optimized query"
131
  assert result["display_query"] == "What is the effect of AI on human rights?"
132
  assert result["original_query"] == "AI human rights"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  assert result["search_query"] == "keyword optimized query"
131
  assert result["display_query"] == "What is the effect of AI on human rights?"
132
  assert result["original_query"] == "AI human rights"
133
+
134
+
135
+ def test_parse_qu_result_with_route_and_sql():
136
+ from src.query_engine import _parse_qu_result
137
+ raw = '{"action": "search", "route": "sql", "search_query": "PTS China", "display_query": "PTS scores for China?", "sql_query": "SELECT * FROM t", "reasoning": "data lookup"}'
138
+ result = _parse_qu_result(raw, "original")
139
+ assert result["route"] == "sql"
140
+ assert result["sql_query"] == "SELECT * FROM t"
141
+
142
+
143
+ def test_parse_qu_result_defaults_route_to_vector():
144
+ from src.query_engine import _parse_qu_result
145
+ raw = '{"action": "search", "search_query": "test", "display_query": "test?"}'
146
+ result = _parse_qu_result(raw, "original")
147
+ assert result["route"] == "vector"
148
+ assert result.get("sql_query") is None
149
+
150
+
151
+ def test_load_sql_schema_missing_file():
152
+ from src.query_engine import _load_sql_schema_summary
153
+ result = _load_sql_schema_summary("/nonexistent/path/sql_db")
154
+ assert result == ""
155
+
156
+
157
+ def test_load_sql_schema_valid_file(tmp_path):
158
+ import json
159
+ from src.query_engine import _load_sql_schema_summary
160
+ schema_dir = tmp_path / "sql_db"
161
+ schema_dir.mkdir()
162
+ schema = {
163
+ "test_table": {
164
+ "source_file": "data.csv",
165
+ "columns": [{"name": "x", "type": "TEXT", "sample": ["a"]}],
166
+ "row_count": 10,
167
+ }
168
+ }
169
+ (schema_dir / "sql_schemas.json").write_text(json.dumps(schema))
170
+ result = _load_sql_schema_summary(str(schema_dir))
171
+ assert "test_table" in result
172
+ assert "10 rows" in result