Files changed (3) hide show
  1. .gitignore +1 -2
  2. README.md +1 -1
  3. inference.py +166 -365
.gitignore CHANGED
@@ -30,5 +30,4 @@ Thumbs.db
30
  # Pytest / coverage
31
  .pytest_cache/
32
  .coverage
33
- htmlcov/
34
- ai_env/
 
30
  # Pytest / coverage
31
  .pytest_cache/
32
  .coverage
33
+ htmlcov/
 
README.md CHANGED
@@ -254,4 +254,4 @@ debugging where production data isn't available.
254
  `run_partial` + `ask_clarification` before `submit_query` models real
255
  developer behaviour: check syntax, check schema, then commit. The turn
256
  penalty in the reward function ensures the agent learns to use these
257
- efficiently rather than exhausting the budget.
 
254
  `run_partial` + `ask_clarification` before `submit_query` models real
255
  developer behaviour: check syntax, check schema, then commit. The turn
256
  penalty in the reward function ensures the agent learns to use these
257
+ efficiently rather than exhausting the budget.
inference.py CHANGED
@@ -1,425 +1,226 @@
1
  """
2
- Inference Script — Interactive SQL Query Repair Environment
3
- ===========================================================
 
4
 
5
- STDOUT FORMAT
 
6
  [START] task=<task_id> env=sql-repair model=<model_name>
7
  [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
8
- [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
9
-
10
- ENV VARS
11
- HF_TOKEN / API_KEY — LLM API key
12
- API_BASE_URL — LLM endpoint (default: HF router)
13
- MODEL_NAME — Model ID (default: Qwen2.5-72B-Instruct)
14
- ENV_SERVER_URL — Running server (default: http://localhost:8000)
15
- SQL_TASK_ID — Task to run (default: easy_001)
16
- SQL_DIFFICULTY — Difficulty (default: easy)
17
  """
18
 
19
- import json
 
 
20
  import os
21
- import sys
22
  import textwrap
23
  from typing import List, Optional
24
 
25
- import httpx
26
  from openai import OpenAI
27
 
28
- # ── Environment configuration ─────────────────────────────────────────────────
29
- API_KEY: str = os.environ.get("API_KEY", "")
30
- API_BASE_URL: str = os.environ.get("API_BASE_URL", "")
31
- MODEL_NAME: str = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")
32
- ENV_SERVER_URL: str = os.getenv("ENV_SERVER_URL", "http://localhost:8000").rstrip("/")
33
-
34
- TASK_ID: str = os.getenv("SQL_TASK_ID", "easy_001")
35
- DIFFICULTY: str = os.getenv("SQL_DIFFICULTY", "easy")
36
- BENCHMARK: str = "sql-repair"
37
-
38
- # Max turns per difficulty (mirrors env rules)
39
- MAX_TURNS_MAP = {"easy": 3, "medium": 5, "hard": 8}
40
-
41
- TEMPERATURE = 0.3 # Low temp → more deterministic SQL generation
42
- MAX_TOKENS = 512
43
- SUCCESS_SCORE_THRESHOLD = 0.5 # score in [0, 1]
44
-
45
- # ── Prompts ───────────────────────────────────────────────────────────────────
46
- SYSTEM_PROMPT = textwrap.dedent("""
47
- You are an expert SQL debugger. Your job is to fix a broken SQL query.
48
-
49
- DATABASE SCHEMA (SQLite):
50
- users (id, name, email, age, city, active)
51
- orders (id, user_id, product_id, amount, created_at)
52
- products (id, name, price, category, stock)
53
- reviews (id, user_id, product_id, rating, comment)
54
- tags (id, product_id, tag)
55
-
56
- RULES:
57
- - You have a limited number of turns. Use them wisely.
58
- - You have THREE action types:
59
- 1. ask_clarification — ask ONE focused question about schema or intent.
60
- Use ONLY when the intent is genuinely ambiguous.
61
- 2. run_partial — test a candidate query to see if it runs without error.
62
- Use to validate your fix before submitting.
63
- 3. submit_query — your final corrected query. Ends the episode.
64
-
65
- STRATEGY:
66
- - Read the broken query and the intent carefully.
67
- - Identify the exact bug (wrong table name, wrong operator, bad join key, etc.)
68
- - For easy tasks: fix directly and submit.
69
- - For medium/hard tasks: run_partial once to verify, then submit.
70
- - Only ask_clarification if you truly cannot determine the schema/intent.
71
- - Never submit a query you have not at least mentally validated.
72
-
73
- RESPONSE FORMAT — respond with a JSON object and nothing else.
74
- Do NOT wrap in markdown fences.
75
-
76
- To ask a clarification:
77
- {"action": "ask_clarification", "question": "<your question>"}
78
-
79
- To test a query:
80
- {"action": "run_partial", "query": "<SELECT ...>"}
81
-
82
- To submit your final fix:
83
- {"action": "submit_query", "query": "<SELECT ...>"}
84
- """).strip()
85
-
86
-
87
- def build_user_prompt(obs: dict, history: List[str]) -> str:
88
- history_block = "\n".join(history) if history else "None"
89
- clarification = obs.get("clarification_response") or "None"
90
- error_msg = obs.get("error_message") or "None"
91
- return textwrap.dedent(f"""
92
- TASK ID: {obs.get('task_id', TASK_ID)}
93
- DIFFICULTY: {obs.get('difficulty', DIFFICULTY)}
94
- INTENT: {obs.get('intent', '')}
95
- SCHEMA HINT: {obs.get('schema_hint', '')}
96
- BROKEN QUERY: {obs.get('broken_query', '')}
97
- TURN: {obs.get('turn', '?')} / {obs.get('max_turns', '?')}
98
- LAST ERROR: {error_msg}
99
- LAST CLARIFICATION RESPONSE: {clarification}
100
- LAST ACTION TYPE: {obs.get('last_action_type') or 'None'}
101
-
102
- HISTORY:
103
- {history_block}
104
-
105
- Respond with a JSON object for your next action.
106
- """).strip()
107
-
108
 
109
- # ── Logging ───────────────────────────────────────────────────────────────────
110
  def log_start(task: str, env: str, model: str) -> None:
111
  print(f"[START] task={task} env={env} model={model}", flush=True)
112
 
113
 
114
  def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
115
- error_val = error if error else "null"
116
- action_clean = action.replace("\n", " ").replace("\r", "")[:200]
117
  print(
118
- f"[STEP] step={step} action={action_clean} reward={reward:.2f} "
119
- f"done={str(done).lower()} error={error_val}",
120
  flush=True,
121
  )
122
 
123
 
124
  def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
125
- rewards_str = ",".join(f"{r:.2f}" for r in rewards)
126
  print(
127
  f"[END] success={str(success).lower()} steps={steps} "
128
- f"score={score:.3f} rewards={rewards_str}",
129
  flush=True,
130
  )
131
 
 
 
 
132
 
133
- # ── HTTP helpers ──────────────────────────────────────────────────────────────
134
- def api_reset(http: httpx.Client, task_id: str, difficulty: str) -> dict:
135
- try:
136
- print(f"[DEBUG] Resetting environment: {ENV_SERVER_URL}/reset", flush=True)
137
- resp = http.post(
138
- f"{ENV_SERVER_URL}/reset",
139
- json={"task_id": task_id, "difficulty": difficulty},
140
- timeout=30,
141
- )
142
- resp.raise_for_status()
143
- print(f"[DEBUG] Reset successful", flush=True)
144
- return resp.json()
145
- except httpx.TimeoutException as exc:
146
- print(f"[DEBUG] /reset timed out: {exc}", flush=True)
147
- raise
148
- except httpx.HTTPStatusError as exc:
149
- print(f"[DEBUG] /reset HTTP {exc.response.status_code}: {exc.response.text[:300]}", flush=True)
150
- raise
151
- except httpx.RequestError as exc:
152
- print(f"[DEBUG] /reset connection error: {exc}", flush=True)
153
- raise
154
- except Exception as exc:
155
- print(f"[DEBUG] /reset unexpected error: {exc}", flush=True)
156
- raise
157
-
158
-
159
- def api_step(http: httpx.Client, payload: dict) -> dict:
160
- try:
161
- resp = http.post(
162
- f"{ENV_SERVER_URL}/step",
163
- json=payload,
164
- timeout=30,
165
- )
166
- resp.raise_for_status()
167
- return resp.json()
168
- except httpx.TimeoutException as exc:
169
- print(f"[DEBUG] /step timed out: {exc}", flush=True)
170
- raise
171
- except httpx.HTTPStatusError as exc:
172
- print(f"[DEBUG] /step HTTP {exc.response.status_code}: {exc.response.text[:300]}", flush=True)
173
- raise
174
- except httpx.RequestError as exc:
175
- print(f"[DEBUG] /step connection error: {exc}", flush=True)
176
- raise
177
- except Exception as exc:
178
- print(f"[DEBUG] /step unexpected error: {exc}", flush=True)
179
- raise
180
 
181
 
182
- def api_grader(http: httpx.Client) -> dict:
183
- try:
184
- resp = http.get(f"{ENV_SERVER_URL}/grader", timeout=30)
185
- resp.raise_for_status()
186
- return resp.json()
187
- except httpx.TimeoutException as exc:
188
- print(f"[DEBUG] /grader timed out: {exc}", flush=True)
189
- raise
190
- except httpx.HTTPStatusError as exc:
191
- print(f"[DEBUG] /grader HTTP {exc.response.status_code}: {exc.response.text[:300]}", flush=True)
192
- raise
193
- except httpx.RequestError as exc:
194
- print(f"[DEBUG] /grader connection error: {exc}", flush=True)
195
- raise
196
- except Exception as exc:
197
- print(f"[DEBUG] /grader unexpected error: {exc}", flush=True)
198
- raise
199
 
 
 
 
200
 
201
- # ── LLM call ─────────────────────────────────────────────────────────────────
202
- def get_model_action(llm: OpenAI, obs: dict, history: List[str]) -> dict:
203
- """
204
- Ask the LLM what action to take.
205
- Always returns a valid action dict — never raises.
206
- Falls back to submitting the broken query on any failure.
207
- """
208
- fallback = {"action": "submit_query", "query": obs.get("broken_query", "SELECT 1")}
 
209
 
210
- # Build prompt
211
- try:
212
- user_prompt = build_user_prompt(obs, history)
213
- except Exception as exc:
214
- print(f"[DEBUG] prompt build error: {exc}", flush=True)
215
- return fallback
216
 
217
- print(f"[DEBUG] Calling LLM: {MODEL_NAME} at {API_BASE_URL}", flush=True)
218
-
219
- # Call LLM
220
  try:
221
- completion = llm.chat.completions.create(
222
  model=MODEL_NAME,
223
  messages=[
224
  {"role": "system", "content": SYSTEM_PROMPT},
225
- {"role": "user", "content": user_prompt},
226
  ],
227
  temperature=TEMPERATURE,
228
  max_tokens=MAX_TOKENS,
229
- stream=False,
230
  )
231
- print(f"[DEBUG] LLM call successful", flush=True)
232
  except Exception as exc:
233
- print(f"[DEBUG] LLM API call failed: {exc}", flush=True)
234
- return fallback
235
 
236
- # Extract text
237
- try:
238
- raw = (completion.choices[0].message.content or "").strip()
239
- except Exception as exc:
240
- print(f"[DEBUG] LLM response read failed: {exc}", flush=True)
241
- return fallback
242
 
243
- # Strip markdown fences if present
244
- try:
245
- if raw.startswith("```"):
246
- parts = raw.split("```")
247
- raw = parts[1] if len(parts) > 1 else raw
248
- if raw.startswith("json"):
249
- raw = raw[4:]
250
- raw = raw.strip()
251
- except Exception as exc:
252
- print(f"[DEBUG] fence strip failed: {exc}", flush=True)
253
 
254
- # Parse JSON
255
- try:
256
- parsed = json.loads(raw)
257
- except json.JSONDecodeError as exc:
258
- print(f"[DEBUG] JSON parse failed ({exc}): {raw[:200]}", flush=True)
259
- return fallback
260
 
261
- # Validate action field
262
- try:
263
- action = parsed.get("action", "")
264
- if action not in ("ask_clarification", "run_partial", "submit_query"):
265
- print(f"[DEBUG] unknown action from model: {action!r}", flush=True)
266
- return fallback
267
- except Exception as exc:
268
- print(f"[DEBUG] action validation error: {exc}", flush=True)
269
- return fallback
270
-
271
- return parsed
272
-
273
-
274
- def action_to_payload(parsed: dict) -> dict:
275
- """Convert LLM response dict to the JSON shape /step expects."""
276
- action = parsed.get("action", "")
277
- if action == "submit_query":
278
- return {"type": "submit_query", "query": parsed.get("query", "")}
279
- elif action == "run_partial":
280
- return {"type": "run_partial", "query": parsed.get("query", "")}
281
- elif action == "ask_clarification":
282
- return {"type": "ask_clarification", "question": parsed.get("question", "")}
283
- # Safe fallback (should not be reached after validation)
284
- return {"type": "submit_query", "query": parsed.get("query", "SELECT 1")}
285
-
286
-
287
- def action_repr(parsed: dict) -> str:
288
- """One-line string for [STEP] logging."""
289
- action = parsed.get("action", "unknown")
290
- if action in ("submit_query", "run_partial"):
291
- q = parsed.get("query", "").replace("\n", " ")
292
- return f'{action}("{q}")'
293
- elif action == "ask_clarification":
294
- return f'ask_clarification("{parsed.get("question", "")}")'
295
- return action
296
-
297
-
298
- # ── Main ──────────────────────────────────────────────────────────────────────
299
- def main() -> None:
300
- # Validate environment variables first
301
- print(f"[DEBUG] Checking environment variables...", flush=True)
302
- print(f"[DEBUG] API_KEY present: {bool(API_KEY)}", flush=True)
303
- print(f"[DEBUG] API_BASE_URL: {API_BASE_URL[:50] if API_BASE_URL else 'MISSING'}...", flush=True)
304
- print(f"[DEBUG] ENV_SERVER_URL: {ENV_SERVER_URL}", flush=True)
305
- print(f"[DEBUG] MODEL_NAME: {MODEL_NAME}", flush=True)
306
-
307
- if not API_KEY or not API_BASE_URL:
308
- print("[ERROR] Missing required environment variables: API_KEY and API_BASE_URL must be set", flush=True)
309
- # Still emit start/end for validator
310
- log_start(task=TASK_ID, env=BENCHMARK, model=MODEL_NAME)
311
- log_end(success=False, steps=0, score=0.0, rewards=[])
312
- sys.exit(1)
313
-
314
- rewards: List[float] = []
315
- steps_taken = 0
316
- score = 0.0
317
- success = False
318
-
319
- # Emit [START] before anything that can fail so the validator always sees it
320
- log_start(task=TASK_ID, env=BENCHMARK, model=MODEL_NAME)
321
 
322
- # Initialise LLM client
323
- try:
324
- print(f"[DEBUG] Initializing OpenAI client with base_url={API_BASE_URL}", flush=True)
325
- llm = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
326
- print(f"[DEBUG] OpenAI client initialized successfully", flush=True)
327
- except Exception as exc:
328
- print(f"[DEBUG] OpenAI client init failed: {exc}", flush=True)
329
- log_end(success=False, steps=0, score=0.0, rewards=[])
330
- sys.exit(1)
331
 
332
- max_turns = MAX_TURNS_MAP.get(DIFFICULTY, 5)
 
 
333
  history: List[str] = []
334
 
 
335
  try:
336
- with httpx.Client() as http:
337
-
338
- # ── Reset ─────────────────────────────────────────────────────
339
- try:
340
- obs: dict = api_reset(http, TASK_ID, DIFFICULTY)
341
- except Exception as exc:
342
- print(f"[DEBUG] Fatal: could not reset environment: {exc}", flush=True)
343
- log_end(success=False, steps=0, score=0.0, rewards=[])
344
- return
345
-
346
- # ── Episode loop ───────────────────────────────────────────────
347
- for step in range(1, max_turns + 1):
348
- print(f"[DEBUG] Starting step {step}/{max_turns}", flush=True)
349
-
350
- # Get action from LLM (never raises)
351
- parsed = get_model_action(llm, obs, history)
352
-
353
- try:
354
- payload = action_to_payload(parsed)
355
- action_str = action_repr(parsed)
356
- except Exception as exc:
357
- print(f"[DEBUG] payload build error at step {step}: {exc}", flush=True)
358
- payload = {"type": "submit_query", "query": obs.get("broken_query", "SELECT 1")}
359
- action_str = "submit_query(fallback)"
360
-
361
- # Send action to environment
362
- try:
363
- step_result = api_step(http, payload)
364
- except Exception as exc:
365
- print(f"[DEBUG] /step failed at step {step}: {exc}", flush=True)
366
- # Log a failed step and end the episode
367
- log_step(step=step, action=action_str, reward=0.0, done=True, error=str(exc))
368
- rewards.append(0.0)
369
- steps_taken = step
370
- break
371
-
372
- # Parse step result
373
- try:
374
- reward: float = float(step_result.get("reward") or 0.0)
375
- done: bool = bool(step_result.get("done", False))
376
- info: dict = step_result.get("info") or {}
377
- new_obs = step_result.get("observation")
378
- if isinstance(new_obs, dict):
379
- obs = new_obs
380
- except Exception as exc:
381
- print(f"[DEBUG] step result parse error: {exc}", flush=True)
382
- reward, done, info = 0.0, True, {}
383
-
384
- error_msg: Optional[str] = obs.get("error_message") if isinstance(obs, dict) else None
385
- feedback: str = info.get("feedback", "") if isinstance(info, dict) else ""
386
-
387
- rewards.append(reward)
388
- steps_taken = step
389
-
390
- log_step(step=step, action=action_str, reward=reward, done=done, error=error_msg)
391
-
392
- history.append(
393
- f"Step {step} [{parsed.get('action', '?')}]: {action_str} "
394
- f"-> reward={reward:+.2f} | feedback: {feedback}"
395
- )
396
-
397
- if done:
398
- print(f"[DEBUG] Episode done at step {step}", flush=True)
399
- break
400
-
401
- # ── Grader ────────────────────────────────────────────────────
402
- try:
403
- print(f"[DEBUG] Getting final score from grader", flush=True)
404
- grader_result = api_grader(http)
405
- score = float(grader_result.get("score", 0.0))
406
- score = min(max(score, 0.0), 1.0)
407
- print(f"[DEBUG] Grader score: {score}", flush=True)
408
- except Exception as exc:
409
- print(f"[DEBUG] /grader failed, falling back to reward sum: {exc}", flush=True)
410
- score = min(max(sum(rewards), 0.0), 1.0)
411
-
412
- success = score >= SUCCESS_SCORE_THRESHOLD
413
 
414
  except Exception as exc:
415
- print(f"[DEBUG] Unhandled episode error: {exc}", flush=True)
416
- import traceback
417
- traceback.print_exc()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
 
419
- finally:
420
- print(f"[DEBUG] Episode complete. Success: {success}, Steps: {steps_taken}, Score: {score}", flush=True)
421
- log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
422
 
423
 
424
  if __name__ == "__main__":
425
- main()
 
1
  """
2
+ inference.py
3
+ ============
4
+ Baseline inference script for the Interactive SQL Query Repair environment.
5
 
6
+ Mandatory stdout format
7
+ -----------------------
8
  [START] task=<task_id> env=sql-repair model=<model_name>
9
  [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
10
+ [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...>
 
 
 
 
 
 
 
 
11
  """
12
 
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
  import os
 
17
  import textwrap
18
  from typing import List, Optional
19
 
 
20
  from openai import OpenAI
21
 
22
+ # ---------------------------------------------------------------------------
23
+ # Config — exactly matching competition sample pattern
24
+ # ---------------------------------------------------------------------------
25
+
26
+ API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
27
+ API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
28
+ MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
29
+ SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")
30
+ BENCHMARK = "sql-repair"
31
+
32
+ EVAL_TASKS = [
33
+ ("easy_001", "easy"),
34
+ ("easy_007", "easy"),
35
+ ("medium_001", "medium"),
36
+ ("medium_005", "medium"),
37
+ ("hard_001", "hard"),
38
+ ("hard_004", "hard"),
39
+ ]
40
+
41
+ MAX_STEPS = 8
42
+ TEMPERATURE = 0.2
43
+ MAX_TOKENS = 400
44
+
45
+ # ---------------------------------------------------------------------------
46
+ # Logging helpers (mandatory stdout format)
47
+ # ---------------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
 
 
49
  def log_start(task: str, env: str, model: str) -> None:
50
  print(f"[START] task={task} env={env} model={model}", flush=True)
51
 
52
 
53
  def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
54
+ action_clean = action.replace("\n", " ").replace("\r", "")
 
55
  print(
56
+ f"[STEP] step={step} action={action_clean!r} "
57
+ f"reward={reward:.2f} done={str(done).lower()} error={error or 'null'}",
58
  flush=True,
59
  )
60
 
61
 
62
  def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
 
63
  print(
64
  f"[END] success={str(success).lower()} steps={steps} "
65
+ f"score={score:.3f} rewards={','.join(f'{r:.2f}' for r in rewards)}",
66
  flush=True,
67
  )
68
 
69
+ # ---------------------------------------------------------------------------
70
+ # HTTP helpers
71
+ # ---------------------------------------------------------------------------
72
 
73
+ def _post(path: str, body: dict) -> dict:
74
+ import urllib.request, json
75
+ url = SERVER_URL.rstrip("/") + path
76
+ req = urllib.request.Request(
77
+ url, json.dumps(body).encode(),
78
+ headers={"Content-Type": "application/json"}
79
+ )
80
+ with urllib.request.urlopen(req, timeout=30) as r:
81
+ return json.loads(r.read())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
 
83
 
84
+ def _get(path: str) -> dict:
85
+ import urllib.request, json
86
+ with urllib.request.urlopen(SERVER_URL.rstrip("/") + path, timeout=30) as r:
87
+ return json.loads(r.read())
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
+ # ---------------------------------------------------------------------------
90
+ # LLM helpers
91
+ # ---------------------------------------------------------------------------
92
 
93
+ SYSTEM_PROMPT = textwrap.dedent("""
94
+ You are an expert SQL repair agent.
95
+ Fix the broken SQL query so it satisfies the given intent.
96
+ Rules:
97
+ - SQLite-compatible SQL only.
98
+ - No window functions — use subqueries or CTEs instead.
99
+ - Output ONLY the fixed SQL, no explanation, no markdown, no backticks.
100
+ - For clarification requests, output ONLY the question text.
101
+ """).strip()
102
 
 
 
 
 
 
 
103
 
104
+ def _ask_llm(client: OpenAI, user_prompt: str) -> str:
 
 
105
  try:
106
+ resp = client.chat.completions.create(
107
  model=MODEL_NAME,
108
  messages=[
109
  {"role": "system", "content": SYSTEM_PROMPT},
110
+ {"role": "user", "content": user_prompt},
111
  ],
112
  temperature=TEMPERATURE,
113
  max_tokens=MAX_TOKENS,
 
114
  )
115
+ return (resp.choices[0].message.content or "").strip()
116
  except Exception as exc:
117
+ print(f"[DEBUG] LLM error: {exc}", flush=True)
118
+ return ""
119
 
 
 
 
 
 
 
120
 
121
+ def repair_prompt(intent, schema, broken, clarif, error, history) -> str:
122
+ return textwrap.dedent(f"""
123
+ Intent : {intent}
124
+ Schema : {schema}
125
+ Broken query : {broken}
126
+ Clarification: {clarif or 'None'}
127
+ Last error : {error or 'None'}
128
+ History : {chr(10).join(history[-4:]) or 'None'}
129
+ Output the corrected SQL query only.
130
+ """).strip()
131
 
 
 
 
 
 
 
132
 
133
+ def clarify_prompt(intent, schema, broken) -> str:
134
+ return textwrap.dedent(f"""
135
+ Intent : {intent}
136
+ Schema : {schema}
137
+ Broken query : {broken}
138
+ Ask ONE clarification question to help fix this query.
139
+ Output ONLY the question text.
140
+ """).strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
+ # ---------------------------------------------------------------------------
143
+ # Episode runner
144
+ # ---------------------------------------------------------------------------
 
 
 
 
 
 
145
 
146
+ def run_episode(client: OpenAI, task_id: str, difficulty: str) -> float:
147
+ rewards: List[float] = []
148
+ steps_taken, score, success = 0, 0.0, False
149
  history: List[str] = []
150
 
151
+ log_start(task_id, BENCHMARK, MODEL_NAME)
152
  try:
153
+ obs = _post("/reset", {"task_id": task_id, "difficulty": difficulty})
154
+ intent = obs["intent"]
155
+ schema = obs["schema_hint"]
156
+ broken_q = obs["broken_query"]
157
+ clarif_resp: Optional[str] = None
158
+ last_error: Optional[str] = None
159
+
160
+ for step in range(1, MAX_STEPS + 1):
161
+ steps_taken = step
162
+
163
+ if step == 1:
164
+ q = _ask_llm(client, clarify_prompt(intent, schema, broken_q)) \
165
+ or "What are the correct table and column names?"
166
+ action_body = {"type": "ask_clarification", "question": q}
167
+ action_str = f"ask_clarification({q!r})"
168
+ elif step == 2:
169
+ fq = _ask_llm(client, repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
170
+ action_body = {"type": "run_partial", "query": fq}
171
+ action_str = f"run_partial({fq!r})"
172
+ else:
173
+ fq = _ask_llm(client, repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
174
+ action_body = {"type": "submit_query", "query": fq}
175
+ action_str = f"submit_query({fq!r})"
176
+
177
+ result = _post("/step", action_body)
178
+ step_obs = result["observation"]
179
+ reward = float(result["reward"])
180
+ done = bool(result["done"])
181
+ info = result["info"]
182
+ err = step_obs.get("error_message")
183
+
184
+ rewards.append(reward)
185
+ clarif_resp = step_obs.get("clarification_response")
186
+ last_error = err
187
+ history.append(
188
+ f"step={step} type={action_body['type']} "
189
+ f"reward={reward:+.2f} feedback={info['feedback']!r}"
190
+ )
191
+
192
+ log_step(step, action_str, reward, done, err)
193
+ if done:
194
+ break
195
+
196
+ g = _get("/grader")
197
+ score = float(g["score"])
198
+ success = bool(g["solved"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
200
  except Exception as exc:
201
+ print(f"[DEBUG] Episode error: {exc}", flush=True)
202
+
203
+ log_end(success, steps_taken, score, rewards)
204
+ return score
205
+
206
+ # ---------------------------------------------------------------------------
207
+ # Main — async to match competition sample pattern
208
+ # ---------------------------------------------------------------------------
209
+
210
+ async def main() -> None:
211
+ client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
212
+
213
+ scores: List[float] = []
214
+ for task_id, difficulty in EVAL_TASKS:
215
+ try:
216
+ scores.append(run_episode(client, task_id, difficulty))
217
+ except Exception as exc:
218
+ print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True)
219
+ scores.append(0.0)
220
 
221
+ avg = sum(scores) / len(scores) if scores else 0.0
222
+ print(f"\n[SUMMARY] tasks={len(scores)} avg_score={avg:.3f}", flush=True)
 
223
 
224
 
225
  if __name__ == "__main__":
226
+ asyncio.run(main())