VaiVas commited on
Commit
96b0235
·
verified ·
1 Parent(s): e0b7d34

Update inference.py

Browse files
Files changed (1) hide show
  1. inference.py +284 -205
inference.py CHANGED
@@ -1,205 +1,284 @@
1
- """
2
- Inference Script for Code Migration Benchmark
3
- =============================================
4
- Follows the mandatory OpenENV STDOUT format:
5
- [START] task=<task_name> env=<benchmark> model=<model_name>
6
- [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
7
- [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
8
- """
9
-
10
- import os
11
- import json
12
- import httpx
13
- import asyncio
14
- from typing import List, Optional
15
- from openai import OpenAI
16
-
17
- # ---------- Configuration ----------
18
-
19
- API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
20
- MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o")
21
- API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY")
22
-
23
- # The environment URL provided by OpenENV (usually 7860 or 8000)
24
- ENV_URL = os.getenv("ENV_URL", "http://localhost:7860")
25
- BENCHMARK = "CodeMigration-v1"
26
- TASKS = ["task_easy", "task_medium", "task_hard"]
27
-
28
- MAX_STEPS = 15
29
- SUCCESS_THRESHOLD = 0.5
30
-
31
-
32
- # ---------- Structured Logging ----------
33
-
34
-
35
- def log_start(task: str, env: str, model: str) -> None:
36
- print(f"[START] task={task} env={env} model={model}", flush=True)
37
-
38
- def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
39
- error_val = error if error else "null"
40
- done_val = str(done).lower()
41
- # Action summary to keep log line single-row
42
- action_clean = action.replace("\n", "\\n")[:100]
43
- print(
44
- f"[STEP] step={step} action={action_clean} reward={reward:.2f} done={done_val} error={error_val}",
45
- flush=True,
46
- )
47
-
48
- def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
49
- rewards_str = ",".join(f"{r:.2f}" for r in rewards)
50
- success_val = str(success).lower()
51
- # Note: score uses .3f per sample requirement
52
- print(f"[END] success={success_val} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
53
-
54
- # ---------- Environment Client ----------
55
-
56
- class EnvClient:
57
- """HTTP client for the Code Migration FastAPI environment."""
58
- def __init__(self, base_url: str):
59
- self.base_url = base_url.rstrip("/")
60
- self.client = httpx.Client(timeout=60.0)
61
- self.session_id = None
62
-
63
- def reset(self, task_id: str) -> dict:
64
- resp = self.client.post(f"{self.base_url}/reset", params={"task_id": task_id})
65
- resp.raise_for_status()
66
- data = resp.json()
67
- self.session_id = data["session_id"]
68
- return data["observation"]
69
-
70
- def step(self, action_dict: dict) -> dict:
71
- resp = self.client.post(
72
- f"{self.base_url}/step",
73
- params={"session_id": self.session_id},
74
- json=action_dict
75
- )
76
- resp.raise_for_status()
77
- return resp.json()
78
-
79
- def grade(self) -> dict:
80
- resp = self.client.get(f"{self.base_url}/grader", params={"session_id": self.session_id})
81
- resp.raise_for_status()
82
- return resp.json()
83
-
84
- def close(self):
85
- self.client.close()
86
-
87
- # ---------- Agent Logic (Tavily + OpenAI) ----------
88
-
89
- class MigrationAgent:
90
- def __init__(self, client: OpenAI):
91
- self.client = client
92
- self.lookup_cache = {}
93
-
94
- def find_equivalent(self, old_call: str, target_lib: str) -> str:
95
- cache_key = f"{old_call}->{target_lib}"
96
- if cache_key in self.lookup_cache:
97
- return self.lookup_cache[cache_key]
98
-
99
- query = f"Python: what is the {target_lib} equivalent of {old_call}?"
100
- try:
101
- # Using the MCP tool to perform search
102
- resp = self.client.responses.create(
103
- model=MODEL_NAME,
104
- tools=[{
105
- "type": "mcp",
106
- "server_label": "tavily",
107
- "server_url": "https://mcp.tavily.com/mcp/?tavilyApiKey=" + os.getenv("TAVILY_API_KEY", ""),
108
- "require_approval": "never"
109
- }],
110
- input=[
111
- {"role": "system", "content": "Return JSON: {'equivalent_function': 'name'}"},
112
- {"role": "user", "content": query},
113
- ],
114
- )
115
- data = json.loads(resp.output_text.strip().replace("```json", "").replace("```", ""))
116
- res = data.get("equivalent_function", old_call)
117
- self.lookup_cache[cache_key] = res
118
- return res
119
- except:
120
- return old_call
121
-
122
- def rewrite(self, code: str, replacements: dict, target_lib: str) -> str:
123
- mapping = "\n".join([f"- {k} -> {v}" for k, v in replacements.items()])
124
- prompt = f"Migrate to {target_lib}. Mapping:\n{mapping}\n\nCode:\n{code}"
125
-
126
- response = self.client.chat.completions.create(
127
- model=MODEL_NAME,
128
- messages=[
129
- {"role": "system", "content": "Refactor code. Return ONLY valid python code. No markdown."},
130
- {"role": "user", "content": prompt}
131
- ],
132
- temperature=0.0
133
- )
134
- return response.choices[0].message.content.strip().replace("```python", "").replace("```", "")
135
-
136
- # ---------- Main Loop ----------
137
-
138
- async def run_task(client: OpenAI, env: EnvClient, task_id: str):
139
- agent = MigrationAgent(client)
140
-
141
- log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
142
-
143
- rewards = []
144
- steps_taken = 0
145
- success = False
146
- score = 0.0
147
-
148
- try:
149
- obs = env.reset(task_id)
150
-
151
- for step_num in range(1, MAX_STEPS + 1):
152
- pending = obs.get("pending_functions", [])
153
- if not pending:
154
- break
155
-
156
- # Agent processing
157
- target_lib = obs.get("target_library", "unknown")
158
- task_func = pending[0]
159
- old_calls = task_func.get("library_functions_used", [])
160
-
161
- replacements = {call: agent.find_equivalent(call, target_lib) for call in old_calls}
162
- new_code = agent.rewrite(task_func["code"], replacements, target_lib)
163
-
164
- # Env step
165
- action_dict = {"func_id": task_func["func_id"], "rewritten_code": new_code}
166
- result = env.step(action_dict)
167
-
168
- obs = result["observation"]
169
- reward = result["reward"]["value"]
170
- done = result["done"]
171
- error = obs.get("last_error")
172
-
173
- rewards.append(reward)
174
- steps_taken = step_num
175
-
176
- log_step(step=step_num, action=json.dumps(replacements), reward=reward, done=done, error=error)
177
-
178
- if done:
179
- break
180
-
181
- grade = env.grade()
182
- score = grade.get("score", 0.0)
183
- success = grade.get("success", False)
184
-
185
- except Exception as e:
186
- print(f"[DEBUG] Error running {task_id}: {e}", flush=True)
187
- finally:
188
- log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
189
-
190
- async def main():
191
- if not API_KEY:
192
- print("[ERROR] Missing API key.", flush=True)
193
- return
194
-
195
- client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
196
- env = EnvClient(ENV_URL)
197
-
198
- try:
199
- for task_id in TASKS:
200
- await run_task(client, env, task_id)
201
- finally:
202
- env.close()
203
-
204
- if __name__ == "__main__":
205
- asyncio.run(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ inference.py — Code Migration OpenEnv Agent
4
+ ===========================================
5
+ Runs an LLM agent across migration tasks and emits structured stdout logs.
6
+
7
+ Stdout format (must not deviate):
8
+ [START] task=<task> env=<benchmark> model=<model>
9
+ [STEP] step=<n> action=<action> reward=<0.00> done=<true|false> error=<msg|null>
10
+ [END] success=<true|false> steps=<n> score=<0.000> rewards=<r1,r2,...>
11
+ """
12
+
13
+ import os
14
+ import json
15
+ from typing import List, Optional, Dict
16
+
17
+ from dotenv import load_dotenv
18
+ load_dotenv()
19
+
20
+ import httpx
21
+ from openai import OpenAI
22
+
23
+ # ---------------------------------------------------------------------------
24
+ # Configuration
25
+ # ---------------------------------------------------------------------------
26
+
27
+ API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
28
+ MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o")
29
+ API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY")
30
+
31
+ ENV_BASE_URL = os.getenv("ENV_URL", "http://localhost:7860")
32
+ BENCHMARK = "CodeMigration-v1"
33
+
34
+ TAVILY_API_KEY = os.getenv("TAVILY_API_KEY", "")
35
+
36
+ MAX_STEPS = 15
37
+ SUCCESS_THRESHOLD = 0.5
38
+
39
+ TASKS = [
40
+ "task_easy",
41
+ "task_medium",
42
+ "task_hard",
43
+ ]
44
+
45
+ # ---------------------------------------------------------------------------
46
+ # Logging (STRICT FORMAT)
47
+ # ---------------------------------------------------------------------------
48
+
49
+ def log_start(task: str, env: str, model: str) -> None:
50
+ print(f"[START] task={task} env={env} model={model}", flush=True)
51
+
52
+
53
+ def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
54
+ error_val = error if error else "null"
55
+ done_val = str(done).lower()
56
+
57
+ action_clean = action.replace("\n", "\\n")[:100]
58
+
59
+ print(
60
+ f"[STEP] step={step} action={action_clean} reward={reward:.2f} done={done_val} error={error_val}",
61
+ flush=True,
62
+ )
63
+
64
+
65
+ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
66
+ rewards_str = ",".join(f"{r:.2f}" for r in rewards)
67
+
68
+ print(
69
+ f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}",
70
+ flush=True,
71
+ )
72
+
73
+ # ---------------------------------------------------------------------------
74
+ # Environment Client
75
+ # ---------------------------------------------------------------------------
76
+
77
+ class CodeMigrationClient:
78
+ def __init__(self, base_url: str):
79
+ self.base_url = base_url.rstrip("/")
80
+ self.client = httpx.Client(timeout=60.0)
81
+ self.session_id: Optional[str] = None
82
+
83
+ def reset(self, task_id: str) -> dict:
84
+ r = self.client.post(f"{self.base_url}/reset", params={"task_id": task_id})
85
+ r.raise_for_status()
86
+ data = r.json()
87
+ self.session_id = data["session_id"]
88
+ return data["observation"]
89
+
90
+ def step(self, action: Dict) -> dict:
91
+ r = self.client.post(
92
+ f"{self.base_url}/step",
93
+ params={"session_id": self.session_id},
94
+ json=action,
95
+ )
96
+ r.raise_for_status()
97
+ return r.json()
98
+
99
+ def grade(self) -> dict:
100
+ r = self.client.get(
101
+ f"{self.base_url}/grader",
102
+ params={"session_id": self.session_id},
103
+ )
104
+ r.raise_for_status()
105
+ return r.json()
106
+
107
+ def close(self):
108
+ self.client.close()
109
+
110
+ # ---------------------------------------------------------------------------
111
+ # Agent Logic (WITH TAVILY)
112
+ # ---------------------------------------------------------------------------
113
+
114
+ class MigrationAgent:
115
+ def __init__(self, client: OpenAI):
116
+ self.client = client
117
+ self.cache: Dict[str, str] = {}
118
+
119
+ def find_equivalent(self, old_call: str, target_lib: str) -> str:
120
+ key = f"{old_call}->{target_lib}"
121
+ if key in self.cache:
122
+ return self.cache[key]
123
+
124
+ query = f"Python: what is the {target_lib} equivalent of {old_call}?"
125
+
126
+ try:
127
+ resp = self.client.responses.create(
128
+ model=MODEL_NAME,
129
+ tools=[{
130
+ "type": "mcp",
131
+ "server_label": "tavily",
132
+ "server_url": f"https://mcp.tavily.com/mcp/?tavilyApiKey={TAVILY_API_KEY}",
133
+ "require_approval": "never"
134
+ }],
135
+ input=[
136
+ {"role": "system", "content": "Return JSON: {'equivalent_function': 'name'}"},
137
+ {"role": "user", "content": query},
138
+ ],
139
+ )
140
+
141
+ text = resp.output_text.strip()
142
+ text = text.replace("```json", "").replace("```", "")
143
+
144
+ data = json.loads(text)
145
+ result = data.get("equivalent_function", old_call)
146
+
147
+ self.cache[key] = result
148
+ return result
149
+
150
+ except Exception:
151
+ # Fallback: direct LLM (no tool)
152
+ try:
153
+ resp = self.client.chat.completions.create(
154
+ model=MODEL_NAME,
155
+ messages=[{
156
+ "role": "user",
157
+ "content": f"What is the {target_lib} equivalent of {old_call}? Return only function name."
158
+ }],
159
+ temperature=0.0,
160
+ )
161
+ result = resp.choices[0].message.content.strip()
162
+ self.cache[key] = result
163
+ return result
164
+ except:
165
+ return old_call
166
+
167
+ def rewrite(self, code: str, mapping: Dict[str, str], target_lib: str) -> str:
168
+ mapping_str = "\n".join([f"{k} -> {v}" for k, v in mapping.items()])
169
+
170
+ prompt = (
171
+ f"Rewrite this Python code to use {target_lib}.\n\n"
172
+ f"Mappings:\n{mapping_str}\n\n"
173
+ f"Return ONLY valid Python code.\n\n"
174
+ f"{code}"
175
+ )
176
+
177
+ try:
178
+ resp = self.client.chat.completions.create(
179
+ model=MODEL_NAME,
180
+ messages=[{"role": "user", "content": prompt}],
181
+ temperature=0.0,
182
+ )
183
+ return (
184
+ resp.choices[0].message.content
185
+ .replace("```python", "")
186
+ .replace("```", "")
187
+ .strip()
188
+ )
189
+ except:
190
+ return code
191
+
192
+ # ---------------------------------------------------------------------------
193
+ # Task Runner
194
+ # ---------------------------------------------------------------------------
195
+
196
+ def run_task(client: OpenAI, env: CodeMigrationClient, task_name: str) -> None:
197
+ agent = MigrationAgent(client)
198
+
199
+ rewards: List[float] = []
200
+ steps_taken = 0
201
+ success = False
202
+ score = 0.0
203
+
204
+ log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)
205
+
206
+ try:
207
+ obs = env.reset(task_name)
208
+
209
+ for step in range(1, MAX_STEPS + 1):
210
+ pending = obs.get("pending_functions", [])
211
+ if not pending:
212
+ break
213
+
214
+ func = pending[0]
215
+ target_lib = obs.get("target_library", "")
216
+
217
+ old_calls = func.get("library_functions_used", [])
218
+
219
+ mapping = {
220
+ call: agent.find_equivalent(call, target_lib)
221
+ for call in old_calls
222
+ }
223
+
224
+ new_code = agent.rewrite(func["code"], mapping, target_lib)
225
+
226
+ try:
227
+ result = env.step({
228
+ "func_id": func["func_id"],
229
+ "rewritten_code": new_code,
230
+ })
231
+ error = None
232
+ except Exception as exc:
233
+ error = str(exc)
234
+ break
235
+
236
+ obs = result["observation"]
237
+ reward = result["reward"]["value"]
238
+ done = result["done"]
239
+
240
+ rewards.append(reward)
241
+ steps_taken = step
242
+
243
+ log_step(
244
+ step=step,
245
+ action=json.dumps(mapping),
246
+ reward=reward,
247
+ done=done,
248
+ error=error,
249
+ )
250
+
251
+ if done:
252
+ break
253
+
254
+ grade = env.grade()
255
+ score = grade.get("score", 0.0)
256
+ success = grade.get("success", False)
257
+
258
+ except Exception as exc:
259
+ print(f"[DEBUG] Task {task_name} failed: {exc}", flush=True)
260
+
261
+ finally:
262
+ log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
263
+
264
+ # ---------------------------------------------------------------------------
265
+ # Main
266
+ # ---------------------------------------------------------------------------
267
+
268
+ def main() -> None:
269
+ if not API_KEY:
270
+ print("[ERROR] Missing API key.", flush=True)
271
+ return
272
+
273
+ client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
274
+ env = CodeMigrationClient(ENV_BASE_URL)
275
+
276
+ try:
277
+ for task in TASKS:
278
+ run_task(client, env, task)
279
+ finally:
280
+ env.close()
281
+
282
+
283
+ if __name__ == "__main__":
284
+ main()