sql-repair-env / inference.py
ashoka0402
sample exp 5
ef720ae
Raw History Blame
8.14 kB
"""
inference.py
============
Baseline inference script for the Interactive SQL Query Repair environment.
Mandatory stdout format
-----------------------
[START] task=<task_id> env=sql-repair model=<model_name>
[STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
[END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...>
"""
from __future__ import annotations
import asyncio
import os
import textwrap
from typing import List, Optional
from openai import OpenAI
# ---------------------------------------------------------------------------
# Config — exactly matching competition sample pattern
# ---------------------------------------------------------------------------
API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")
BENCHMARK = "sql-repair"
EVAL_TASKS = [
("easy_001", "easy"),
("easy_007", "easy"),
("medium_001", "medium"),
("medium_005", "medium"),
("hard_001", "hard"),
("hard_004", "hard"),
]
MAX_STEPS = 8
TEMPERATURE = 0.2
MAX_TOKENS = 400
# ---------------------------------------------------------------------------
# Logging helpers (mandatory stdout format)
# ---------------------------------------------------------------------------
def log_start(task: str, env: str, model: str) -> None:
print(f"[START] task={task} env={env} model={model}", flush=True)
def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
action_clean = action.replace("\n", " ").replace("\r", "")
print(
f"[STEP] step={step} action={action_clean!r} "
f"reward={reward:.2f} done={str(done).lower()} error={error or 'null'}",
flush=True,
)
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
print(
f"[END] success={str(success).lower()} steps={steps} "
f"score={score:.3f} rewards={','.join(f'{r:.2f}' for r in rewards)}",
flush=True,
)
# ---------------------------------------------------------------------------
# HTTP helpers
# ---------------------------------------------------------------------------
def _post(path: str, body: dict) -> dict:
import urllib.request, json
url = SERVER_URL.rstrip("/") + path
req = urllib.request.Request(
url, json.dumps(body).encode(),
headers={"Content-Type": "application/json"}
)
with urllib.request.urlopen(req, timeout=30) as r:
return json.loads(r.read())
def _get(path: str) -> dict:
import urllib.request, json
with urllib.request.urlopen(SERVER_URL.rstrip("/") + path, timeout=30) as r:
return json.loads(r.read())
# ---------------------------------------------------------------------------
# LLM helpers
# ---------------------------------------------------------------------------
SYSTEM_PROMPT = textwrap.dedent("""
You are an expert SQL repair agent.
Fix the broken SQL query so it satisfies the given intent.
Rules:
- SQLite-compatible SQL only.
- No window functions — use subqueries or CTEs instead.
- Output ONLY the fixed SQL, no explanation, no markdown, no backticks.
- For clarification requests, output ONLY the question text.
""").strip()
def _ask_llm(client: OpenAI, user_prompt: str) -> str:
try:
resp = client.chat.completions.create(
model=MODEL_NAME,
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_prompt},
],
temperature=TEMPERATURE,
max_tokens=MAX_TOKENS,
)
return (resp.choices[0].message.content or "").strip()
except Exception as exc:
print(f"[DEBUG] LLM error: {exc}", flush=True)
return ""
def repair_prompt(intent, schema, broken, clarif, error, history) -> str:
return textwrap.dedent(f"""
Intent : {intent}
Schema : {schema}
Broken query : {broken}
Clarification: {clarif or 'None'}
Last error : {error or 'None'}
History : {chr(10).join(history[-4:]) or 'None'}
Output the corrected SQL query only.
""").strip()
def clarify_prompt(intent, schema, broken) -> str:
return textwrap.dedent(f"""
Intent : {intent}
Schema : {schema}
Broken query : {broken}
Ask ONE clarification question to help fix this query.
Output ONLY the question text.
""").strip()
# ---------------------------------------------------------------------------
# Episode runner
# ---------------------------------------------------------------------------
def run_episode(client: OpenAI, task_id: str, difficulty: str) -> float:
rewards: List[float] = []
steps_taken, score, success = 0, 0.0, False
history: List[str] = []
log_start(task_id, BENCHMARK, MODEL_NAME)
try:
obs = _post("/reset", {"task_id": task_id, "difficulty": difficulty})
intent = obs["intent"]
schema = obs["schema_hint"]
broken_q = obs["broken_query"]
clarif_resp: Optional[str] = None
last_error: Optional[str] = None
for step in range(1, MAX_STEPS + 1):
steps_taken = step
if step == 1:
q = _ask_llm(client, clarify_prompt(intent, schema, broken_q)) \
or "What are the correct table and column names?"
action_body = {"type": "ask_clarification", "question": q}
action_str = f"ask_clarification({q!r})"
elif step == 2:
fq = _ask_llm(client, repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
action_body = {"type": "run_partial", "query": fq}
action_str = f"run_partial({fq!r})"
else:
fq = _ask_llm(client, repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
action_body = {"type": "submit_query", "query": fq}
action_str = f"submit_query({fq!r})"
result = _post("/step", action_body)
step_obs = result["observation"]
reward = float(result["reward"])
done = bool(result["done"])
info = result["info"]
err = step_obs.get("error_message")
rewards.append(reward)
clarif_resp = step_obs.get("clarification_response")
last_error = err
history.append(
f"step={step} type={action_body['type']} "
f"reward={reward:+.2f} feedback={info['feedback']!r}"
)
log_step(step, action_str, reward, done, err)
if done:
break
g = _get("/grader")
score = float(g["score"])
success = bool(g["solved"])
except Exception as exc:
print(f"[DEBUG] Episode error: {exc}", flush=True)
log_end(success, steps_taken, score, rewards)
return score
# ---------------------------------------------------------------------------
# Main — async to match competition sample pattern
# ---------------------------------------------------------------------------
async def main() -> None:
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
scores: List[float] = []
for task_id, difficulty in EVAL_TASKS:
try:
scores.append(run_episode(client, task_id, difficulty))
except Exception as exc:
print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True)
scores.append(0.0)
avg = sum(scores) / len(scores) if scores else 0.0
print(f"\n[SUMMARY] tasks={len(scores)} avg_score={avg:.3f}", flush=True)
if __name__ == "__main__":
asyncio.run(main())