ashoka0402 commited on
Commit
44f5d04
·
1 Parent(s): 887fd28

major changes exp4

Browse files
Files changed (1) hide show
  1. inference.py +77 -119
inference.py CHANGED
@@ -8,15 +8,6 @@ Mandatory stdout format
8
  [START] task=<task_id> env=sql-repair model=<model_name>
9
  [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
10
  [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...>
11
-
12
- Strategy (3-phase per episode)
13
- -------------------------------
14
- Turn 1 : ask_clarification about the join key / column names
15
- Turn 2 : run_partial with LLM-repaired query to check for syntax errors
16
- Turn 3+: submit_query with the LLM's best repair
17
-
18
- The LLM is given: intent, schema, broken_query, error feedback, and
19
- clarification answers accumulated so far.
20
  """
21
 
22
  from __future__ import annotations
@@ -24,16 +15,15 @@ from __future__ import annotations
24
  import os
25
  import textwrap
26
  from typing import List, Optional
27
-
28
  from openai import OpenAI
29
 
30
  # ---------------------------------------------------------------------------
31
- # Config — all from environment variables per competition spec
32
  # ---------------------------------------------------------------------------
33
 
34
  API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
35
  MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")
36
- API_KEY = os.getenv("API_KEY", "no-key") # validator injects real key at runtime
37
 
38
  SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")
39
  BENCHMARK = "sql-repair"
@@ -51,6 +41,15 @@ MAX_STEPS = 8
51
  TEMPERATURE = 0.2
52
  MAX_TOKENS = 400
53
 
 
 
 
 
 
 
 
 
 
54
  # ---------------------------------------------------------------------------
55
  # Logging helpers (mandatory stdout format)
56
  # ---------------------------------------------------------------------------
@@ -60,44 +59,40 @@ def log_start(task: str, env: str, model: str) -> None:
60
 
61
 
62
  def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
63
- error_val = error if error else "null"
64
- done_val = str(done).lower()
65
  action_clean = action.replace("\n", " ").replace("\r", "")
66
  print(
67
  f"[STEP] step={step} action={action_clean!r} "
68
- f"reward={reward:.2f} done={done_val} error={error_val}",
69
  flush=True,
70
  )
71
 
72
 
73
  def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
74
- rewards_str = ",".join(f"{r:.2f}" for r in rewards)
75
  print(
76
  f"[END] success={str(success).lower()} steps={steps} "
77
- f"score={score:.3f} rewards={rewards_str}",
78
  flush=True,
79
  )
80
 
81
-
82
  # ---------------------------------------------------------------------------
83
  # HTTP helpers
84
  # ---------------------------------------------------------------------------
85
 
86
  def _post(path: str, body: dict) -> dict:
87
  import urllib.request, json
88
- url = SERVER_URL.rstrip("/") + path
89
- data = json.dumps(body).encode()
90
- req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"})
91
- with urllib.request.urlopen(req, timeout=30) as resp:
92
- return json.loads(resp.read())
 
 
93
 
94
 
95
  def _get(path: str) -> dict:
96
  import urllib.request, json
97
- url = SERVER_URL.rstrip("/") + path
98
- with urllib.request.urlopen(url, timeout=30) as resp:
99
- return json.loads(resp.read())
100
-
101
 
102
  # ---------------------------------------------------------------------------
103
  # LLM helpers
@@ -105,20 +100,19 @@ def _get(path: str) -> dict:
105
 
106
  SYSTEM_PROMPT = textwrap.dedent("""
107
  You are an expert SQL repair agent.
108
- Your job is to fix a broken SQL query so that it correctly satisfies a given intent.
109
-
110
  Rules:
111
- - Use only standard SQLite-compatible SQL.
112
- - Do NOT use window functions (ROW_NUMBER, RANK, etc.) — use subqueries or CTEs instead.
113
- - Output ONLY the fixed SQL query — no explanation, no markdown, no backticks.
114
- - If you are asked to produce a clarification question, output ONLY the question text.
115
  """).strip()
116
 
117
 
118
- def _ask_llm(client: OpenAI, user_prompt: str) -> str:
119
  """Call the LLM via the injected proxy. Returns empty string on error."""
120
  try:
121
- completion = client.chat.completions.create(
122
  model=MODEL_NAME,
123
  messages=[
124
  {"role": "system", "content": SYSTEM_PROMPT},
@@ -126,67 +120,46 @@ def _ask_llm(client: OpenAI, user_prompt: str) -> str:
126
  ],
127
  temperature=TEMPERATURE,
128
  max_tokens=MAX_TOKENS,
129
- stream=False,
130
  )
131
- return (completion.choices[0].message.content or "").strip()
132
  except Exception as exc:
133
- print(f"[DEBUG] LLM call failed: {exc}", flush=True)
134
  return ""
135
 
136
 
137
- def build_repair_prompt(
138
- intent: str,
139
- schema_hint: str,
140
- broken_query: str,
141
- clarification: Optional[str],
142
- last_error: Optional[str],
143
- history: List[str],
144
- ) -> str:
145
- hist_block = "\n".join(history[-4:]) if history else "None"
146
  return textwrap.dedent(f"""
147
- Task intent : {intent}
148
- Schema : {schema_hint}
149
- Broken query : {broken_query}
150
- Clarification : {clarification or "None"}
151
- Last error : {last_error or "None"}
152
- History :
153
- {hist_block}
154
-
155
- Produce the corrected SQL query only.
156
  """).strip()
157
 
158
 
159
- def build_clarify_prompt(intent: str, schema_hint: str, broken_query: str) -> str:
160
  return textwrap.dedent(f"""
161
- Task intent : {intent}
162
- Schema : {schema_hint}
163
- Broken query : {broken_query}
164
-
165
- Ask ONE short clarification question about the schema or intent
166
- that would help you fix this query.
167
  Output ONLY the question text.
168
  """).strip()
169
 
170
-
171
  # ---------------------------------------------------------------------------
172
- # Single episode runner
173
  # ---------------------------------------------------------------------------
174
 
175
- def run_episode(client: OpenAI, task_id: str, difficulty: str) -> tuple[float, int, List[float]]:
176
- """Run one episode. Returns (score, steps_taken, rewards_list)."""
177
- rewards: List[float] = []
178
- steps_taken: int = 0
179
- score: float = 0.0
180
- success: bool = False
181
- history: List[str] = []
182
-
183
- log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
184
-
185
  try:
186
- obs_raw = _post("/reset", {"task_id": task_id, "difficulty": difficulty})
187
- intent = obs_raw["intent"]
188
- schema_hint = obs_raw["schema_hint"]
189
- broken_q = obs_raw["broken_query"]
190
  clarif_resp: Optional[str] = None
191
  last_error: Optional[str] = None
192
 
@@ -194,77 +167,62 @@ def run_episode(client: OpenAI, task_id: str, difficulty: str) -> tuple[float, i
194
  steps_taken = step
195
 
196
  if step == 1:
197
- question = _ask_llm(client, build_clarify_prompt(intent, schema_hint, broken_q)) \
198
  or "What are the correct table and column names?"
199
- action_body = {"type": "ask_clarification", "question": question}
200
- action_str = f"ask_clarification({question!r})"
201
-
202
  elif step == 2:
203
- fixed_q = _ask_llm(client, build_repair_prompt(intent, schema_hint, broken_q, clarif_resp, last_error, history)) \
204
- or broken_q
205
- action_body = {"type": "run_partial", "query": fixed_q}
206
- action_str = f"run_partial({fixed_q!r})"
207
-
208
  else:
209
- fixed_q = _ask_llm(client, build_repair_prompt(intent, schema_hint, broken_q, clarif_resp, last_error, history)) \
210
- or broken_q
211
- action_body = {"type": "submit_query", "query": fixed_q}
212
- action_str = f"submit_query({fixed_q!r})"
213
 
214
  result = _post("/step", action_body)
215
- obs = result["observation"]
216
  reward = float(result["reward"])
217
  done = bool(result["done"])
218
  info = result["info"]
219
- error = obs.get("error_message")
220
 
221
  rewards.append(reward)
222
- clarif_resp = obs.get("clarification_response")
223
- last_error = error
224
-
225
  history.append(
226
- f"Step {step} [{action_body['type']}]: "
227
  f"reward={reward:+.2f} feedback={info['feedback']!r}"
228
  )
229
 
230
- log_step(step=step, action=action_str, reward=reward, done=done, error=error)
231
-
232
  if done:
233
  break
234
 
235
- grade = _get("/grader")
236
- score = float(grade["score"])
237
- success = bool(grade["solved"])
238
 
239
  except Exception as exc:
240
  print(f"[DEBUG] Episode error: {exc}", flush=True)
241
 
242
- log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
243
- return score, steps_taken, rewards
244
-
245
 
246
  # ---------------------------------------------------------------------------
247
  # Main
248
  # ---------------------------------------------------------------------------
249
 
250
  def main() -> None:
251
- client = OpenAI(
252
- base_url=API_BASE_URL,
253
- api_key=API_KEY,
254
- )
255
-
256
- all_scores: List[float] = []
257
-
258
  for task_id, difficulty in EVAL_TASKS:
259
  try:
260
- score, steps, rewards = run_episode(client, task_id, difficulty)
261
  except Exception as exc:
262
  print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True)
263
- score = 0.0
264
- all_scores.append(score)
265
-
266
- avg = sum(all_scores) / len(all_scores) if all_scores else 0.0
267
- print(f"\n[SUMMARY] tasks={len(all_scores)} avg_score={avg:.3f}", flush=True)
268
 
269
 
270
  if __name__ == "__main__":
 
8
  [START] task=<task_id> env=sql-repair model=<model_name>
9
  [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
10
  [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...>
 
 
 
 
 
 
 
 
 
11
  """
12
 
13
  from __future__ import annotations
 
15
  import os
16
  import textwrap
17
  from typing import List, Optional
 
18
  from openai import OpenAI
19
 
20
  # ---------------------------------------------------------------------------
21
+ # Config — exactly as specified in competition docs
22
  # ---------------------------------------------------------------------------
23
 
24
  API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
25
  MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")
26
+ HF_TOKEN = os.getenv("HF_TOKEN") # no default — injected at runtime
27
 
28
  SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")
29
  BENCHMARK = "sql-repair"
 
41
  TEMPERATURE = 0.2
42
  MAX_TOKENS = 400
43
 
44
+ # ---------------------------------------------------------------------------
45
+ # OpenAI client — initialised once using HF_TOKEN as the API key
46
+ # ---------------------------------------------------------------------------
47
+
48
+ client = OpenAI(
49
+ base_url=API_BASE_URL,
50
+ api_key=HF_TOKEN,
51
+ )
52
+
53
  # ---------------------------------------------------------------------------
54
  # Logging helpers (mandatory stdout format)
55
  # ---------------------------------------------------------------------------
 
59
 
60
 
61
  def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
 
 
62
  action_clean = action.replace("\n", " ").replace("\r", "")
63
  print(
64
  f"[STEP] step={step} action={action_clean!r} "
65
+ f"reward={reward:.2f} done={str(done).lower()} error={error or 'null'}",
66
  flush=True,
67
  )
68
 
69
 
70
  def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
 
71
  print(
72
  f"[END] success={str(success).lower()} steps={steps} "
73
+ f"score={score:.3f} rewards={','.join(f'{r:.2f}' for r in rewards)}",
74
  flush=True,
75
  )
76
 
 
77
  # ---------------------------------------------------------------------------
78
  # HTTP helpers
79
  # ---------------------------------------------------------------------------
80
 
81
  def _post(path: str, body: dict) -> dict:
82
  import urllib.request, json
83
+ url = SERVER_URL.rstrip("/") + path
84
+ req = urllib.request.Request(
85
+ url, json.dumps(body).encode(),
86
+ headers={"Content-Type": "application/json"}
87
+ )
88
+ with urllib.request.urlopen(req, timeout=30) as r:
89
+ return json.loads(r.read())
90
 
91
 
92
  def _get(path: str) -> dict:
93
  import urllib.request, json
94
+ with urllib.request.urlopen(SERVER_URL.rstrip("/") + path, timeout=30) as r:
95
+ return json.loads(r.read())
 
 
96
 
97
  # ---------------------------------------------------------------------------
98
  # LLM helpers
 
100
 
101
  SYSTEM_PROMPT = textwrap.dedent("""
102
  You are an expert SQL repair agent.
103
+ Fix the broken SQL query so it satisfies the given intent.
 
104
  Rules:
105
+ - SQLite-compatible SQL only.
106
+ - No window functions — use subqueries or CTEs instead.
107
+ - Output ONLY the fixed SQL, no explanation, no markdown, no backticks.
108
+ - For clarification requests, output ONLY the question text.
109
  """).strip()
110
 
111
 
112
+ def _ask_llm(user_prompt: str) -> str:
113
  """Call the LLM via the injected proxy. Returns empty string on error."""
114
  try:
115
+ resp = client.chat.completions.create(
116
  model=MODEL_NAME,
117
  messages=[
118
  {"role": "system", "content": SYSTEM_PROMPT},
 
120
  ],
121
  temperature=TEMPERATURE,
122
  max_tokens=MAX_TOKENS,
 
123
  )
124
+ return (resp.choices[0].message.content or "").strip()
125
  except Exception as exc:
126
+ print(f"[DEBUG] LLM error: {exc}", flush=True)
127
  return ""
128
 
129
 
130
+ def repair_prompt(intent, schema, broken, clarif, error, history) -> str:
 
 
 
 
 
 
 
 
131
  return textwrap.dedent(f"""
132
+ Intent : {intent}
133
+ Schema : {schema}
134
+ Broken query : {broken}
135
+ Clarification: {clarif or 'None'}
136
+ Last error : {error or 'None'}
137
+ History : {chr(10).join(history[-4:]) or 'None'}
138
+ Output the corrected SQL query only.
 
 
139
  """).strip()
140
 
141
 
142
+ def clarify_prompt(intent, schema, broken) -> str:
143
  return textwrap.dedent(f"""
144
+ Intent : {intent}
145
+ Schema : {schema}
146
+ Broken query : {broken}
147
+ Ask ONE clarification question to help fix this query.
 
 
148
  Output ONLY the question text.
149
  """).strip()
150
 
 
151
  # ---------------------------------------------------------------------------
152
+ # Episode runner
153
  # ---------------------------------------------------------------------------
154
 
155
+ def run_episode(task_id: str, difficulty: str) -> float:
156
+ rewards, steps_taken, score, success, history = [], 0, 0.0, False, []
157
+ log_start(task_id, BENCHMARK, MODEL_NAME)
 
 
 
 
 
 
 
158
  try:
159
+ obs = _post("/reset", {"task_id": task_id, "difficulty": difficulty})
160
+ intent = obs["intent"]
161
+ schema = obs["schema_hint"]
162
+ broken_q = obs["broken_query"]
163
  clarif_resp: Optional[str] = None
164
  last_error: Optional[str] = None
165
 
 
167
  steps_taken = step
168
 
169
  if step == 1:
170
+ q = _ask_llm(clarify_prompt(intent, schema, broken_q)) \
171
  or "What are the correct table and column names?"
172
+ action_body = {"type": "ask_clarification", "question": q}
173
+ action_str = f"ask_clarification({q!r})"
 
174
  elif step == 2:
175
+ fq = _ask_llm(repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
176
+ action_body = {"type": "run_partial", "query": fq}
177
+ action_str = f"run_partial({fq!r})"
 
 
178
  else:
179
+ fq = _ask_llm(repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
180
+ action_body = {"type": "submit_query", "query": fq}
181
+ action_str = f"submit_query({fq!r})"
 
182
 
183
  result = _post("/step", action_body)
184
+ step_obs = result["observation"]
185
  reward = float(result["reward"])
186
  done = bool(result["done"])
187
  info = result["info"]
188
+ err = step_obs.get("error_message")
189
 
190
  rewards.append(reward)
191
+ clarif_resp = step_obs.get("clarification_response")
192
+ last_error = err
 
193
  history.append(
194
+ f"step={step} type={action_body['type']} "
195
  f"reward={reward:+.2f} feedback={info['feedback']!r}"
196
  )
197
 
198
+ log_step(step, action_str, reward, done, err)
 
199
  if done:
200
  break
201
 
202
+ g = _get("/grader")
203
+ score = float(g["score"])
204
+ success = bool(g["solved"])
205
 
206
  except Exception as exc:
207
  print(f"[DEBUG] Episode error: {exc}", flush=True)
208
 
209
+ log_end(success, steps_taken, score, rewards)
210
+ return score
 
211
 
212
  # ---------------------------------------------------------------------------
213
  # Main
214
  # ---------------------------------------------------------------------------
215
 
216
  def main() -> None:
217
+ scores = []
 
 
 
 
 
 
218
  for task_id, difficulty in EVAL_TASKS:
219
  try:
220
+ scores.append(run_episode(task_id, difficulty))
221
  except Exception as exc:
222
  print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True)
223
+ scores.append(0.0)
224
+ avg = sum(scores) / len(scores) if scores else 0.0
225
+ print(f"\n[SUMMARY] tasks={len(scores)} avg_score={avg:.3f}", flush=True)
 
 
226
 
227
 
228
  if __name__ == "__main__":