""" inference.py ============ Baseline inference script for the Interactive SQL Query Repair environment. Mandatory stdout format ----------------------- [START] task= env=sql-repair model= [STEP] step= action= reward=<0.00> done= error= [END] success= steps= score= rewards= """ from __future__ import annotations import asyncio import os import textwrap from typing import List, Optional from openai import OpenAI # --------------------------------------------------------------------------- # Config — exactly matching competition sample pattern # --------------------------------------------------------------------------- API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY") API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1" MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct" SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860") BENCHMARK = "sql-repair" EVAL_TASKS = [ ("easy_001", "easy"), ("easy_007", "easy"), ("medium_001", "medium"), ("medium_005", "medium"), ("hard_001", "hard"), ("hard_004", "hard"), ] MAX_STEPS = 8 TEMPERATURE = 0.2 MAX_TOKENS = 400 # --------------------------------------------------------------------------- # Logging helpers (mandatory stdout format) # --------------------------------------------------------------------------- def log_start(task: str, env: str, model: str) -> None: print(f"[START] task={task} env={env} model={model}", flush=True) def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None: action_clean = action.replace("\n", " ").replace("\r", "") print( f"[STEP] step={step} action={action_clean!r} " f"reward={reward:.2f} done={str(done).lower()} error={error or 'null'}", flush=True, ) def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None: print( f"[END] success={str(success).lower()} steps={steps} " f"score={score:.3f} rewards={','.join(f'{r:.2f}' for r in rewards)}", flush=True, ) # --------------------------------------------------------------------------- # HTTP helpers # --------------------------------------------------------------------------- def _post(path: str, body: dict) -> dict: import urllib.request, json url = SERVER_URL.rstrip("/") + path req = urllib.request.Request( url, json.dumps(body).encode(), headers={"Content-Type": "application/json"} ) with urllib.request.urlopen(req, timeout=30) as r: return json.loads(r.read()) def _get(path: str) -> dict: import urllib.request, json with urllib.request.urlopen(SERVER_URL.rstrip("/") + path, timeout=30) as r: return json.loads(r.read()) # --------------------------------------------------------------------------- # LLM helpers # --------------------------------------------------------------------------- SYSTEM_PROMPT = textwrap.dedent(""" You are an expert SQL repair agent. Fix the broken SQL query so it satisfies the given intent. Rules: - SQLite-compatible SQL only. - No window functions — use subqueries or CTEs instead. - Output ONLY the fixed SQL, no explanation, no markdown, no backticks. - For clarification requests, output ONLY the question text. """).strip() def _ask_llm(client: OpenAI, user_prompt: str) -> str: try: resp = client.chat.completions.create( model=MODEL_NAME, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user_prompt}, ], temperature=TEMPERATURE, max_tokens=MAX_TOKENS, ) return (resp.choices[0].message.content or "").strip() except Exception as exc: print(f"[DEBUG] LLM error: {exc}", flush=True) return "" def repair_prompt(intent, schema, broken, clarif, error, history) -> str: return textwrap.dedent(f""" Intent : {intent} Schema : {schema} Broken query : {broken} Clarification: {clarif or 'None'} Last error : {error or 'None'} History : {chr(10).join(history[-4:]) or 'None'} Output the corrected SQL query only. """).strip() def clarify_prompt(intent, schema, broken) -> str: return textwrap.dedent(f""" Intent : {intent} Schema : {schema} Broken query : {broken} Ask ONE clarification question to help fix this query. Output ONLY the question text. """).strip() # --------------------------------------------------------------------------- # Episode runner # --------------------------------------------------------------------------- def run_episode(client: OpenAI, task_id: str, difficulty: str) -> float: rewards: List[float] = [] steps_taken, score, success = 0, 0.0, False history: List[str] = [] log_start(task_id, BENCHMARK, MODEL_NAME) try: obs = _post("/reset", {"task_id": task_id, "difficulty": difficulty}) intent = obs["intent"] schema = obs["schema_hint"] broken_q = obs["broken_query"] clarif_resp: Optional[str] = None last_error: Optional[str] = None for step in range(1, MAX_STEPS + 1): steps_taken = step if step == 1: q = _ask_llm(client, clarify_prompt(intent, schema, broken_q)) \ or "What are the correct table and column names?" action_body = {"type": "ask_clarification", "question": q} action_str = f"ask_clarification({q!r})" elif step == 2: fq = _ask_llm(client, repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q action_body = {"type": "run_partial", "query": fq} action_str = f"run_partial({fq!r})" else: fq = _ask_llm(client, repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q action_body = {"type": "submit_query", "query": fq} action_str = f"submit_query({fq!r})" result = _post("/step", action_body) step_obs = result["observation"] reward = float(result["reward"]) done = bool(result["done"]) info = result["info"] err = step_obs.get("error_message") rewards.append(reward) clarif_resp = step_obs.get("clarification_response") last_error = err history.append( f"step={step} type={action_body['type']} " f"reward={reward:+.2f} feedback={info['feedback']!r}" ) log_step(step, action_str, reward, done, err) if done: break g = _get("/grader") score = float(g["score"]) success = bool(g["solved"]) except Exception as exc: print(f"[DEBUG] Episode error: {exc}", flush=True) log_end(success, steps_taken, score, rewards) return score # --------------------------------------------------------------------------- # Main — async to match competition sample pattern # --------------------------------------------------------------------------- async def main() -> None: client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY) scores: List[float] = [] for task_id, difficulty in EVAL_TASKS: try: scores.append(run_episode(client, task_id, difficulty)) except Exception as exc: print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True) scores.append(0.0) avg = sum(scores) / len(scores) if scores else 0.0 print(f"\n[SUMMARY] tasks={len(scores)} avg_score={avg:.3f}", flush=True) if __name__ == "__main__": asyncio.run(main())