Spaces:
Sleeping
Sleeping
ashoka0402 commited on
Commit ·
44f5d04
1
Parent(s): 887fd28
major changes exp4
Browse files- inference.py +77 -119
inference.py
CHANGED
|
@@ -8,15 +8,6 @@ Mandatory stdout format
|
|
| 8 |
[START] task=<task_id> env=sql-repair model=<model_name>
|
| 9 |
[STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
|
| 10 |
[END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...>
|
| 11 |
-
|
| 12 |
-
Strategy (3-phase per episode)
|
| 13 |
-
-------------------------------
|
| 14 |
-
Turn 1 : ask_clarification about the join key / column names
|
| 15 |
-
Turn 2 : run_partial with LLM-repaired query to check for syntax errors
|
| 16 |
-
Turn 3+: submit_query with the LLM's best repair
|
| 17 |
-
|
| 18 |
-
The LLM is given: intent, schema, broken_query, error feedback, and
|
| 19 |
-
clarification answers accumulated so far.
|
| 20 |
"""
|
| 21 |
|
| 22 |
from __future__ import annotations
|
|
@@ -24,16 +15,15 @@ from __future__ import annotations
|
|
| 24 |
import os
|
| 25 |
import textwrap
|
| 26 |
from typing import List, Optional
|
| 27 |
-
|
| 28 |
from openai import OpenAI
|
| 29 |
|
| 30 |
# ---------------------------------------------------------------------------
|
| 31 |
-
# Config —
|
| 32 |
# ---------------------------------------------------------------------------
|
| 33 |
|
| 34 |
API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
|
| 35 |
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")
|
| 36 |
-
|
| 37 |
|
| 38 |
SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")
|
| 39 |
BENCHMARK = "sql-repair"
|
|
@@ -51,6 +41,15 @@ MAX_STEPS = 8
|
|
| 51 |
TEMPERATURE = 0.2
|
| 52 |
MAX_TOKENS = 400
|
| 53 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
# ---------------------------------------------------------------------------
|
| 55 |
# Logging helpers (mandatory stdout format)
|
| 56 |
# ---------------------------------------------------------------------------
|
|
@@ -60,44 +59,40 @@ def log_start(task: str, env: str, model: str) -> None:
|
|
| 60 |
|
| 61 |
|
| 62 |
def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
|
| 63 |
-
error_val = error if error else "null"
|
| 64 |
-
done_val = str(done).lower()
|
| 65 |
action_clean = action.replace("\n", " ").replace("\r", "")
|
| 66 |
print(
|
| 67 |
f"[STEP] step={step} action={action_clean!r} "
|
| 68 |
-
f"reward={reward:.2f} done={
|
| 69 |
flush=True,
|
| 70 |
)
|
| 71 |
|
| 72 |
|
| 73 |
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
|
| 74 |
-
rewards_str = ",".join(f"{r:.2f}" for r in rewards)
|
| 75 |
print(
|
| 76 |
f"[END] success={str(success).lower()} steps={steps} "
|
| 77 |
-
f"score={score:.3f} rewards={
|
| 78 |
flush=True,
|
| 79 |
)
|
| 80 |
|
| 81 |
-
|
| 82 |
# ---------------------------------------------------------------------------
|
| 83 |
# HTTP helpers
|
| 84 |
# ---------------------------------------------------------------------------
|
| 85 |
|
| 86 |
def _post(path: str, body: dict) -> dict:
|
| 87 |
import urllib.request, json
|
| 88 |
-
url
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
|
|
|
|
|
|
| 93 |
|
| 94 |
|
| 95 |
def _get(path: str) -> dict:
|
| 96 |
import urllib.request, json
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
return json.loads(resp.read())
|
| 100 |
-
|
| 101 |
|
| 102 |
# ---------------------------------------------------------------------------
|
| 103 |
# LLM helpers
|
|
@@ -105,20 +100,19 @@ def _get(path: str) -> dict:
|
|
| 105 |
|
| 106 |
SYSTEM_PROMPT = textwrap.dedent("""
|
| 107 |
You are an expert SQL repair agent.
|
| 108 |
-
|
| 109 |
-
|
| 110 |
Rules:
|
| 111 |
-
-
|
| 112 |
-
-
|
| 113 |
-
- Output ONLY the fixed SQL
|
| 114 |
-
-
|
| 115 |
""").strip()
|
| 116 |
|
| 117 |
|
| 118 |
-
def _ask_llm(
|
| 119 |
"""Call the LLM via the injected proxy. Returns empty string on error."""
|
| 120 |
try:
|
| 121 |
-
|
| 122 |
model=MODEL_NAME,
|
| 123 |
messages=[
|
| 124 |
{"role": "system", "content": SYSTEM_PROMPT},
|
|
@@ -126,67 +120,46 @@ def _ask_llm(client: OpenAI, user_prompt: str) -> str:
|
|
| 126 |
],
|
| 127 |
temperature=TEMPERATURE,
|
| 128 |
max_tokens=MAX_TOKENS,
|
| 129 |
-
stream=False,
|
| 130 |
)
|
| 131 |
-
return (
|
| 132 |
except Exception as exc:
|
| 133 |
-
print(f"[DEBUG] LLM
|
| 134 |
return ""
|
| 135 |
|
| 136 |
|
| 137 |
-
def
|
| 138 |
-
intent: str,
|
| 139 |
-
schema_hint: str,
|
| 140 |
-
broken_query: str,
|
| 141 |
-
clarification: Optional[str],
|
| 142 |
-
last_error: Optional[str],
|
| 143 |
-
history: List[str],
|
| 144 |
-
) -> str:
|
| 145 |
-
hist_block = "\n".join(history[-4:]) if history else "None"
|
| 146 |
return textwrap.dedent(f"""
|
| 147 |
-
|
| 148 |
-
Schema
|
| 149 |
-
Broken query
|
| 150 |
-
Clarification
|
| 151 |
-
Last error
|
| 152 |
-
History
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
Produce the corrected SQL query only.
|
| 156 |
""").strip()
|
| 157 |
|
| 158 |
|
| 159 |
-
def
|
| 160 |
return textwrap.dedent(f"""
|
| 161 |
-
|
| 162 |
-
Schema : {
|
| 163 |
-
Broken query : {
|
| 164 |
-
|
| 165 |
-
Ask ONE short clarification question about the schema or intent
|
| 166 |
-
that would help you fix this query.
|
| 167 |
Output ONLY the question text.
|
| 168 |
""").strip()
|
| 169 |
|
| 170 |
-
|
| 171 |
# ---------------------------------------------------------------------------
|
| 172 |
-
#
|
| 173 |
# ---------------------------------------------------------------------------
|
| 174 |
|
| 175 |
-
def run_episode(
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
steps_taken: int = 0
|
| 179 |
-
score: float = 0.0
|
| 180 |
-
success: bool = False
|
| 181 |
-
history: List[str] = []
|
| 182 |
-
|
| 183 |
-
log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
|
| 184 |
-
|
| 185 |
try:
|
| 186 |
-
|
| 187 |
-
intent =
|
| 188 |
-
|
| 189 |
-
broken_q =
|
| 190 |
clarif_resp: Optional[str] = None
|
| 191 |
last_error: Optional[str] = None
|
| 192 |
|
|
@@ -194,77 +167,62 @@ def run_episode(client: OpenAI, task_id: str, difficulty: str) -> tuple[float, i
|
|
| 194 |
steps_taken = step
|
| 195 |
|
| 196 |
if step == 1:
|
| 197 |
-
|
| 198 |
or "What are the correct table and column names?"
|
| 199 |
-
action_body = {"type": "ask_clarification", "question":
|
| 200 |
-
action_str = f"ask_clarification({
|
| 201 |
-
|
| 202 |
elif step == 2:
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
action_str = f"run_partial({fixed_q!r})"
|
| 207 |
-
|
| 208 |
else:
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
action_str = f"submit_query({fixed_q!r})"
|
| 213 |
|
| 214 |
result = _post("/step", action_body)
|
| 215 |
-
|
| 216 |
reward = float(result["reward"])
|
| 217 |
done = bool(result["done"])
|
| 218 |
info = result["info"]
|
| 219 |
-
|
| 220 |
|
| 221 |
rewards.append(reward)
|
| 222 |
-
clarif_resp =
|
| 223 |
-
last_error =
|
| 224 |
-
|
| 225 |
history.append(
|
| 226 |
-
f"
|
| 227 |
f"reward={reward:+.2f} feedback={info['feedback']!r}"
|
| 228 |
)
|
| 229 |
|
| 230 |
-
log_step(step
|
| 231 |
-
|
| 232 |
if done:
|
| 233 |
break
|
| 234 |
|
| 235 |
-
|
| 236 |
-
score = float(
|
| 237 |
-
success = bool(
|
| 238 |
|
| 239 |
except Exception as exc:
|
| 240 |
print(f"[DEBUG] Episode error: {exc}", flush=True)
|
| 241 |
|
| 242 |
-
log_end(success
|
| 243 |
-
return score
|
| 244 |
-
|
| 245 |
|
| 246 |
# ---------------------------------------------------------------------------
|
| 247 |
# Main
|
| 248 |
# ---------------------------------------------------------------------------
|
| 249 |
|
| 250 |
def main() -> None:
|
| 251 |
-
|
| 252 |
-
base_url=API_BASE_URL,
|
| 253 |
-
api_key=API_KEY,
|
| 254 |
-
)
|
| 255 |
-
|
| 256 |
-
all_scores: List[float] = []
|
| 257 |
-
|
| 258 |
for task_id, difficulty in EVAL_TASKS:
|
| 259 |
try:
|
| 260 |
-
|
| 261 |
except Exception as exc:
|
| 262 |
print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True)
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
avg = sum(all_scores) / len(all_scores) if all_scores else 0.0
|
| 267 |
-
print(f"\n[SUMMARY] tasks={len(all_scores)} avg_score={avg:.3f}", flush=True)
|
| 268 |
|
| 269 |
|
| 270 |
if __name__ == "__main__":
|
|
|
|
| 8 |
[START] task=<task_id> env=sql-repair model=<model_name>
|
| 9 |
[STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
|
| 10 |
[END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
"""
|
| 12 |
|
| 13 |
from __future__ import annotations
|
|
|
|
| 15 |
import os
|
| 16 |
import textwrap
|
| 17 |
from typing import List, Optional
|
|
|
|
| 18 |
from openai import OpenAI
|
| 19 |
|
| 20 |
# ---------------------------------------------------------------------------
|
| 21 |
+
# Config — exactly as specified in competition docs
|
| 22 |
# ---------------------------------------------------------------------------
|
| 23 |
|
| 24 |
API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
|
| 25 |
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")
|
| 26 |
+
HF_TOKEN = os.getenv("HF_TOKEN") # no default — injected at runtime
|
| 27 |
|
| 28 |
SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")
|
| 29 |
BENCHMARK = "sql-repair"
|
|
|
|
| 41 |
TEMPERATURE = 0.2
|
| 42 |
MAX_TOKENS = 400
|
| 43 |
|
| 44 |
+
# ---------------------------------------------------------------------------
|
| 45 |
+
# OpenAI client — initialised once using HF_TOKEN as the API key
|
| 46 |
+
# ---------------------------------------------------------------------------
|
| 47 |
+
|
| 48 |
+
client = OpenAI(
|
| 49 |
+
base_url=API_BASE_URL,
|
| 50 |
+
api_key=HF_TOKEN,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
# ---------------------------------------------------------------------------
|
| 54 |
# Logging helpers (mandatory stdout format)
|
| 55 |
# ---------------------------------------------------------------------------
|
|
|
|
| 59 |
|
| 60 |
|
| 61 |
def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
|
|
|
|
|
|
|
| 62 |
action_clean = action.replace("\n", " ").replace("\r", "")
|
| 63 |
print(
|
| 64 |
f"[STEP] step={step} action={action_clean!r} "
|
| 65 |
+
f"reward={reward:.2f} done={str(done).lower()} error={error or 'null'}",
|
| 66 |
flush=True,
|
| 67 |
)
|
| 68 |
|
| 69 |
|
| 70 |
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
|
|
|
|
| 71 |
print(
|
| 72 |
f"[END] success={str(success).lower()} steps={steps} "
|
| 73 |
+
f"score={score:.3f} rewards={','.join(f'{r:.2f}' for r in rewards)}",
|
| 74 |
flush=True,
|
| 75 |
)
|
| 76 |
|
|
|
|
| 77 |
# ---------------------------------------------------------------------------
|
| 78 |
# HTTP helpers
|
| 79 |
# ---------------------------------------------------------------------------
|
| 80 |
|
| 81 |
def _post(path: str, body: dict) -> dict:
|
| 82 |
import urllib.request, json
|
| 83 |
+
url = SERVER_URL.rstrip("/") + path
|
| 84 |
+
req = urllib.request.Request(
|
| 85 |
+
url, json.dumps(body).encode(),
|
| 86 |
+
headers={"Content-Type": "application/json"}
|
| 87 |
+
)
|
| 88 |
+
with urllib.request.urlopen(req, timeout=30) as r:
|
| 89 |
+
return json.loads(r.read())
|
| 90 |
|
| 91 |
|
| 92 |
def _get(path: str) -> dict:
|
| 93 |
import urllib.request, json
|
| 94 |
+
with urllib.request.urlopen(SERVER_URL.rstrip("/") + path, timeout=30) as r:
|
| 95 |
+
return json.loads(r.read())
|
|
|
|
|
|
|
| 96 |
|
| 97 |
# ---------------------------------------------------------------------------
|
| 98 |
# LLM helpers
|
|
|
|
| 100 |
|
| 101 |
SYSTEM_PROMPT = textwrap.dedent("""
|
| 102 |
You are an expert SQL repair agent.
|
| 103 |
+
Fix the broken SQL query so it satisfies the given intent.
|
|
|
|
| 104 |
Rules:
|
| 105 |
+
- SQLite-compatible SQL only.
|
| 106 |
+
- No window functions — use subqueries or CTEs instead.
|
| 107 |
+
- Output ONLY the fixed SQL, no explanation, no markdown, no backticks.
|
| 108 |
+
- For clarification requests, output ONLY the question text.
|
| 109 |
""").strip()
|
| 110 |
|
| 111 |
|
| 112 |
+
def _ask_llm(user_prompt: str) -> str:
|
| 113 |
"""Call the LLM via the injected proxy. Returns empty string on error."""
|
| 114 |
try:
|
| 115 |
+
resp = client.chat.completions.create(
|
| 116 |
model=MODEL_NAME,
|
| 117 |
messages=[
|
| 118 |
{"role": "system", "content": SYSTEM_PROMPT},
|
|
|
|
| 120 |
],
|
| 121 |
temperature=TEMPERATURE,
|
| 122 |
max_tokens=MAX_TOKENS,
|
|
|
|
| 123 |
)
|
| 124 |
+
return (resp.choices[0].message.content or "").strip()
|
| 125 |
except Exception as exc:
|
| 126 |
+
print(f"[DEBUG] LLM error: {exc}", flush=True)
|
| 127 |
return ""
|
| 128 |
|
| 129 |
|
| 130 |
+
def repair_prompt(intent, schema, broken, clarif, error, history) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
return textwrap.dedent(f"""
|
| 132 |
+
Intent : {intent}
|
| 133 |
+
Schema : {schema}
|
| 134 |
+
Broken query : {broken}
|
| 135 |
+
Clarification: {clarif or 'None'}
|
| 136 |
+
Last error : {error or 'None'}
|
| 137 |
+
History : {chr(10).join(history[-4:]) or 'None'}
|
| 138 |
+
Output the corrected SQL query only.
|
|
|
|
|
|
|
| 139 |
""").strip()
|
| 140 |
|
| 141 |
|
| 142 |
+
def clarify_prompt(intent, schema, broken) -> str:
|
| 143 |
return textwrap.dedent(f"""
|
| 144 |
+
Intent : {intent}
|
| 145 |
+
Schema : {schema}
|
| 146 |
+
Broken query : {broken}
|
| 147 |
+
Ask ONE clarification question to help fix this query.
|
|
|
|
|
|
|
| 148 |
Output ONLY the question text.
|
| 149 |
""").strip()
|
| 150 |
|
|
|
|
| 151 |
# ---------------------------------------------------------------------------
|
| 152 |
+
# Episode runner
|
| 153 |
# ---------------------------------------------------------------------------
|
| 154 |
|
| 155 |
+
def run_episode(task_id: str, difficulty: str) -> float:
|
| 156 |
+
rewards, steps_taken, score, success, history = [], 0, 0.0, False, []
|
| 157 |
+
log_start(task_id, BENCHMARK, MODEL_NAME)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
try:
|
| 159 |
+
obs = _post("/reset", {"task_id": task_id, "difficulty": difficulty})
|
| 160 |
+
intent = obs["intent"]
|
| 161 |
+
schema = obs["schema_hint"]
|
| 162 |
+
broken_q = obs["broken_query"]
|
| 163 |
clarif_resp: Optional[str] = None
|
| 164 |
last_error: Optional[str] = None
|
| 165 |
|
|
|
|
| 167 |
steps_taken = step
|
| 168 |
|
| 169 |
if step == 1:
|
| 170 |
+
q = _ask_llm(clarify_prompt(intent, schema, broken_q)) \
|
| 171 |
or "What are the correct table and column names?"
|
| 172 |
+
action_body = {"type": "ask_clarification", "question": q}
|
| 173 |
+
action_str = f"ask_clarification({q!r})"
|
|
|
|
| 174 |
elif step == 2:
|
| 175 |
+
fq = _ask_llm(repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
|
| 176 |
+
action_body = {"type": "run_partial", "query": fq}
|
| 177 |
+
action_str = f"run_partial({fq!r})"
|
|
|
|
|
|
|
| 178 |
else:
|
| 179 |
+
fq = _ask_llm(repair_prompt(intent, schema, broken_q, clarif_resp, last_error, history)) or broken_q
|
| 180 |
+
action_body = {"type": "submit_query", "query": fq}
|
| 181 |
+
action_str = f"submit_query({fq!r})"
|
|
|
|
| 182 |
|
| 183 |
result = _post("/step", action_body)
|
| 184 |
+
step_obs = result["observation"]
|
| 185 |
reward = float(result["reward"])
|
| 186 |
done = bool(result["done"])
|
| 187 |
info = result["info"]
|
| 188 |
+
err = step_obs.get("error_message")
|
| 189 |
|
| 190 |
rewards.append(reward)
|
| 191 |
+
clarif_resp = step_obs.get("clarification_response")
|
| 192 |
+
last_error = err
|
|
|
|
| 193 |
history.append(
|
| 194 |
+
f"step={step} type={action_body['type']} "
|
| 195 |
f"reward={reward:+.2f} feedback={info['feedback']!r}"
|
| 196 |
)
|
| 197 |
|
| 198 |
+
log_step(step, action_str, reward, done, err)
|
|
|
|
| 199 |
if done:
|
| 200 |
break
|
| 201 |
|
| 202 |
+
g = _get("/grader")
|
| 203 |
+
score = float(g["score"])
|
| 204 |
+
success = bool(g["solved"])
|
| 205 |
|
| 206 |
except Exception as exc:
|
| 207 |
print(f"[DEBUG] Episode error: {exc}", flush=True)
|
| 208 |
|
| 209 |
+
log_end(success, steps_taken, score, rewards)
|
| 210 |
+
return score
|
|
|
|
| 211 |
|
| 212 |
# ---------------------------------------------------------------------------
|
| 213 |
# Main
|
| 214 |
# ---------------------------------------------------------------------------
|
| 215 |
|
| 216 |
def main() -> None:
|
| 217 |
+
scores = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
for task_id, difficulty in EVAL_TASKS:
|
| 219 |
try:
|
| 220 |
+
scores.append(run_episode(task_id, difficulty))
|
| 221 |
except Exception as exc:
|
| 222 |
print(f"[DEBUG] Task {task_id} failed: {exc}", flush=True)
|
| 223 |
+
scores.append(0.0)
|
| 224 |
+
avg = sum(scores) / len(scores) if scores else 0.0
|
| 225 |
+
print(f"\n[SUMMARY] tasks={len(scores)} avg_score={avg:.3f}", flush=True)
|
|
|
|
|
|
|
| 226 |
|
| 227 |
|
| 228 |
if __name__ == "__main__":
|