VaiVas commited on
Commit
a820e84
·
verified ·
1 Parent(s): 01c7be9

Upload 8 files

Browse files
Files changed (8) hide show
  1. Dockerfile +21 -0
  2. inference.py +205 -0
  3. openenv.yaml +25 -0
  4. pyproject.toml +27 -0
  5. requirements.txt +8 -0
  6. server/app.py +111 -0
  7. server/tasks.py +56 -0
  8. uv.lock +0 -0
Dockerfile ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10-slim
2
+
3
+ WORKDIR /app
4
+
5
+ RUN apt-get update && apt-get install -y \
6
+ build-essential \
7
+ git \
8
+ && rm -rf /var/lib/apt/lists/*
9
+
10
+
11
+ COPY requirements.txt .
12
+
13
+ RUN pip install --no-cache-dir -r requirements.txt
14
+
15
+ COPY . .
16
+
17
+ RUN chmod +x inference.py
18
+
19
+ ENV PYTHONUNBUFFERED=1
20
+
21
+ ENTRYPOINT ["python3", "server/app.py"]
inference.py ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Inference Script for Code Migration Benchmark
3
+ =============================================
4
+ Follows the mandatory OpenENV STDOUT format:
5
+ [START] task=<task_name> env=<benchmark> model=<model_name>
6
+ [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
7
+ [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
8
+ """
9
+
10
+ import os
11
+ import json
12
+ import httpx
13
+ import asyncio
14
+ from typing import List, Optional
15
+ from openai import OpenAI
16
+
17
+ # ---------- Configuration ----------
18
+
19
+ API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
20
+ MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o")
21
+ API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY")
22
+
23
+ # The environment URL provided by OpenENV (usually 7860 or 8000)
24
+ ENV_URL = os.getenv("ENV_URL", "http://localhost:7860")
25
+ BENCHMARK = "CodeMigration-v1"
26
+ TASKS = ["task_easy", "task_medium", "task_hard"]
27
+
28
+ MAX_STEPS = 15
29
+ SUCCESS_THRESHOLD = 0.5
30
+
31
+
32
+ # ---------- Structured Logging ----------
33
+
34
+
35
+ def log_start(task: str, env: str, model: str) -> None:
36
+ print(f"[START] task={task} env={env} model={model}", flush=True)
37
+
38
+ def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
39
+ error_val = error if error else "null"
40
+ done_val = str(done).lower()
41
+ # Action summary to keep log line single-row
42
+ action_clean = action.replace("\n", "\\n")[:100]
43
+ print(
44
+ f"[STEP] step={step} action={action_clean} reward={reward:.2f} done={done_val} error={error_val}",
45
+ flush=True,
46
+ )
47
+
48
+ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
49
+ rewards_str = ",".join(f"{r:.2f}" for r in rewards)
50
+ success_val = str(success).lower()
51
+ # Note: score uses .3f per sample requirement
52
+ print(f"[END] success={success_val} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
53
+
54
+ # ---------- Environment Client ----------
55
+
56
+ class EnvClient:
57
+ """HTTP client for the Code Migration FastAPI environment."""
58
+ def __init__(self, base_url: str):
59
+ self.base_url = base_url.rstrip("/")
60
+ self.client = httpx.Client(timeout=60.0)
61
+ self.session_id = None
62
+
63
+ def reset(self, task_id: str) -> dict:
64
+ resp = self.client.post(f"{self.base_url}/reset", params={"task_id": task_id})
65
+ resp.raise_for_status()
66
+ data = resp.json()
67
+ self.session_id = data["session_id"]
68
+ return data["observation"]
69
+
70
+ def step(self, action_dict: dict) -> dict:
71
+ resp = self.client.post(
72
+ f"{self.base_url}/step",
73
+ params={"session_id": self.session_id},
74
+ json=action_dict
75
+ )
76
+ resp.raise_for_status()
77
+ return resp.json()
78
+
79
+ def grade(self) -> dict:
80
+ resp = self.client.get(f"{self.base_url}/grader", params={"session_id": self.session_id})
81
+ resp.raise_for_status()
82
+ return resp.json()
83
+
84
+ def close(self):
85
+ self.client.close()
86
+
87
+ # ---------- Agent Logic (Tavily + OpenAI) ----------
88
+
89
+ class MigrationAgent:
90
+ def __init__(self, client: OpenAI):
91
+ self.client = client
92
+ self.lookup_cache = {}
93
+
94
+ def find_equivalent(self, old_call: str, target_lib: str) -> str:
95
+ cache_key = f"{old_call}->{target_lib}"
96
+ if cache_key in self.lookup_cache:
97
+ return self.lookup_cache[cache_key]
98
+
99
+ query = f"Python: what is the {target_lib} equivalent of {old_call}?"
100
+ try:
101
+ # Using the MCP tool to perform search
102
+ resp = self.client.responses.create(
103
+ model=MODEL_NAME,
104
+ tools=[{
105
+ "type": "mcp",
106
+ "server_label": "tavily",
107
+ "server_url": "https://mcp.tavily.com/mcp/?tavilyApiKey=" + os.getenv("TAVILY_API_KEY", ""),
108
+ "require_approval": "never"
109
+ }],
110
+ input=[
111
+ {"role": "system", "content": "Return JSON: {'equivalent_function': 'name'}"},
112
+ {"role": "user", "content": query},
113
+ ],
114
+ )
115
+ data = json.loads(resp.output_text.strip().replace("```json", "").replace("```", ""))
116
+ res = data.get("equivalent_function", old_call)
117
+ self.lookup_cache[cache_key] = res
118
+ return res
119
+ except:
120
+ return old_call
121
+
122
+ def rewrite(self, code: str, replacements: dict, target_lib: str) -> str:
123
+ mapping = "\n".join([f"- {k} -> {v}" for k, v in replacements.items()])
124
+ prompt = f"Migrate to {target_lib}. Mapping:\n{mapping}\n\nCode:\n{code}"
125
+
126
+ response = self.client.chat.completions.create(
127
+ model=MODEL_NAME,
128
+ messages=[
129
+ {"role": "system", "content": "Refactor code. Return ONLY valid python code. No markdown."},
130
+ {"role": "user", "content": prompt}
131
+ ],
132
+ temperature=0.0
133
+ )
134
+ return response.choices[0].message.content.strip().replace("```python", "").replace("```", "")
135
+
136
+ # ---------- Main Loop ----------
137
+
138
+ async def run_task(client: OpenAI, env: EnvClient, task_id: str):
139
+ agent = MigrationAgent(client)
140
+
141
+ log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
142
+
143
+ rewards = []
144
+ steps_taken = 0
145
+ success = False
146
+ score = 0.0
147
+
148
+ try:
149
+ obs = env.reset(task_id)
150
+
151
+ for step_num in range(1, MAX_STEPS + 1):
152
+ pending = obs.get("pending_functions", [])
153
+ if not pending:
154
+ break
155
+
156
+ # Agent processing
157
+ target_lib = obs.get("target_library", "unknown")
158
+ task_func = pending[0]
159
+ old_calls = task_func.get("library_functions_used", [])
160
+
161
+ replacements = {call: agent.find_equivalent(call, target_lib) for call in old_calls}
162
+ new_code = agent.rewrite(task_func["code"], replacements, target_lib)
163
+
164
+ # Env step
165
+ action_dict = {"func_id": task_func["func_id"], "rewritten_code": new_code}
166
+ result = env.step(action_dict)
167
+
168
+ obs = result["observation"]
169
+ reward = result["reward"]["value"]
170
+ done = result["done"]
171
+ error = obs.get("last_error")
172
+
173
+ rewards.append(reward)
174
+ steps_taken = step_num
175
+
176
+ log_step(step=step_num, action=json.dumps(replacements), reward=reward, done=done, error=error)
177
+
178
+ if done:
179
+ break
180
+
181
+ grade = env.grade()
182
+ score = grade.get("score", 0.0)
183
+ success = grade.get("success", False)
184
+
185
+ except Exception as e:
186
+ print(f"[DEBUG] Error running {task_id}: {e}", flush=True)
187
+ finally:
188
+ log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
189
+
190
+ async def main():
191
+ if not API_KEY:
192
+ print("[ERROR] Missing API key.", flush=True)
193
+ return
194
+
195
+ client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
196
+ env = EnvClient(ENV_URL)
197
+
198
+ try:
199
+ for task_id in TASKS:
200
+ await run_task(client, env, task_id)
201
+ finally:
202
+ env.close()
203
+
204
+ if __name__ == "__main__":
205
+ asyncio.run(main())
openenv.yaml ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ spec_version: 1
2
+ name: code-migration-v1
3
+ type: space
4
+ runtime: fastapi
5
+ app: app:app
6
+ port: 7860
7
+ tasks:
8
+ - id: task_easy
9
+ name: Easy Migration (requests to httpx)
10
+ description: Migrate 2 simple synchronous functions with high API similarity.
11
+ grader:
12
+ endpoint: /grader
13
+ score_range: [0.02, 0.98]
14
+ - id: task_medium
15
+ name: Medium Migration (sqlite3 to sqlalchemy)
16
+ description: Migrate 4 functions involving a shift from raw SQL to an ORM/Core pattern.
17
+ grader:
18
+ endpoint: /grader
19
+ score_range: [0.02, 0.98]
20
+ - id: task_hard
21
+ name: Hard Migration (sync to async httpx)
22
+ description: Migrate 6 functions requiring structural changes for asynchronous execution.
23
+ grader:
24
+ endpoint: /grader
25
+ score_range: [0.02, 0.98]
pyproject.toml ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=68.0", "wheel"]
3
+ build-backend = "setuptools.backends._legacy:_Backend"
4
+
5
+ [project]
6
+ name = "code-migration-v1"
7
+ version = "1.0.0"
8
+ description = "Library migration environment for AI agents using Tavily MCP lookup"
9
+ readme = "README.md"
10
+ license = {text = "MIT"}
11
+ requires-python = ">=3.10"
12
+
13
+ dependencies = [
14
+ "fastapi",
15
+ "uvicorn",
16
+ "pydantic>=2.0",
17
+ "numpy",
18
+ "networkx",
19
+ "openai",
20
+ "httpx",
21
+ "python-dotenv",
22
+ "openenv-core>=0.2.0",
23
+ ]
24
+
25
+ [project.scripts]
26
+ server = "server.app:main"
27
+
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ openenv-core
2
+ pytest
3
+ requests
4
+ httpx
5
+ openai
6
+ pydantic
7
+ python-dotenv
8
+ torch
server/app.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI, HTTPException
2
+ from pydantic import BaseModel, Field
3
+ from typing import Dict, List, Optional, Any
4
+ import ast
5
+ import uuid
6
+ import time
7
+ import threading
8
+ import os
9
+ from tasks import TASKS # Import the tasks we just defined
10
+ import uvicorn
11
+
12
+ class CodeMigrationAction(BaseModel):
13
+ func_id: str
14
+ rewritten_code: str
15
+
16
+ class CodeMigrationObservation(BaseModel):
17
+ pending_functions: List[Dict[str, Any]]
18
+ completed_functions: List[str]
19
+ target_library: str
20
+ last_error: Optional[str] = None
21
+
22
+ class Reward(BaseModel):
23
+ value: float
24
+
25
+ class CodeMigrationEnv:
26
+ def __init__(self, task_config: Dict):
27
+ self.config = task_config
28
+ self.pending = {f["func_id"]: f for f in task_config["functions"]}
29
+ self.completed = []
30
+ self.last_error = None
31
+ self.cumulative_reward = 0.0
32
+
33
+ def _uses_old_library(self, code_str: str) -> bool:
34
+ source_prefix = self.config["source_lib"].split('.')[0]
35
+ try:
36
+ tree = ast.parse(code_str)
37
+ for node in ast.walk(tree):
38
+ if isinstance(node, (ast.Import, ast.ImportFrom)):
39
+ names = [n.name for n in node.names]
40
+ if any(name.startswith(source_prefix) for name in names): return True
41
+ if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith(source_prefix): return True
42
+ return source_prefix in code_str # String fallback for safety
43
+ except: return True
44
+
45
+ def step(self, action: CodeMigrationAction):
46
+ if action.func_id not in self.pending:
47
+ return self.state(), Reward(value=-0.1), False, {"error": "Invalid ID"}
48
+
49
+ try:
50
+ ast.parse(action.rewritten_code)
51
+ if self._uses_old_library(action.rewritten_code):
52
+ reward = -0.2
53
+ self.last_error = f"Still uses {self.config['source_lib']}"
54
+ else:
55
+ reward = 1.0
56
+ self.completed.append(action.func_id)
57
+ del self.pending[action.func_id]
58
+ self.last_error = None
59
+ except SyntaxError as e:
60
+ reward = -0.5
61
+ self.last_error = f"Syntax Error: {e}"
62
+
63
+ self.cumulative_reward += reward
64
+ done = len(self.pending) == 0
65
+ return self.state(), Reward(value=reward), done, {}
66
+
67
+ def state(self) -> CodeMigrationObservation:
68
+ return CodeMigrationObservation(
69
+ pending_functions=list(self.pending.values()),
70
+ completed_functions=self.completed,
71
+ target_library=self.config["target_lib"],
72
+ last_error=self.last_error
73
+ )
74
+
75
+ app = FastAPI()
76
+ sessions: Dict[str, Dict] = {}
77
+ _lock = threading.Lock()
78
+
79
+ @app.post("/reset")
80
+ def reset(task_id: str = "task_easy"):
81
+ if task_id not in TASKS: raise HTTPException(404)
82
+ env = CodeMigrationEnv(TASKS[task_id])
83
+ sid = str(uuid.uuid4())
84
+ with _lock: sessions[sid] = {"env": env, "rewards": []}
85
+ return {"session_id": sid, "observation": env.state().dict()}
86
+
87
+ @app.post("/step")
88
+ def step(session_id: str, action: CodeMigrationAction):
89
+ with _lock:
90
+ if session_id not in sessions: raise HTTPException(404)
91
+ env = sessions[session_id]["env"]
92
+ obs, reward, done, info = env.step(action)
93
+ with _lock: sessions[session_id]["rewards"].append(reward.value)
94
+ return {"observation": obs.dict(), "reward": reward.dict(), "done": done, "info": info}
95
+
96
+ @app.get("/grader")
97
+ def grader(session_id: str):
98
+ with _lock:
99
+ session = sessions.get(session_id)
100
+ if not session: raise HTTPException(404)
101
+ total = sum(session["rewards"])
102
+ n_funcs = len(TASKS[session["env"].config["id"]]["functions"])
103
+ return {"score": max(0.02, min(0.98, total/n_funcs)), "success": len(session["env"].pending) == 0}
104
+
105
+ def main():
106
+ """Entry point for openenv server mode."""
107
+ uvicorn.run(app, host="0.0.0.0", port=7860)
108
+
109
+
110
+ if __name__ == "__main__":
111
+ main()
server/tasks.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+
3
+ def generate_migration_task(difficulty: str = "easy", seed: int = 42):
4
+ """
5
+ Generates a migration task configuration.
6
+
7
+ Easy: Migration between similar APIs (requests -> httpx). 2 functions.
8
+ Medium: Migration involving structural changes (sqlite3 -> SQLAlchemy Core). 4 functions.
9
+ Hard: Migration involving paradigm shifts (Sync requests -> Async httpx with context managers). 6 functions.
10
+ """
11
+ random.seed(seed)
12
+
13
+ if difficulty == "easy":
14
+ source_lib = "requests"
15
+ target_lib = "httpx"
16
+ n_funcs = 2
17
+ template = "def {name}(url):\n import requests\n return requests.{method}(url).json()"
18
+ methods = ["get", "post"]
19
+ elif difficulty == "medium":
20
+ source_lib = "sqlite3"
21
+ target_lib = "sqlalchemy"
22
+ n_funcs = 4
23
+ template = "def {name}(db_path, query):\n import sqlite3\n conn = sqlite3.connect(db_path)\n cur = conn.cursor()\n cur.execute(query)\n return cur.fetchall()"
24
+ methods = ["fetch_all", "get_results", "query_db", "run_sql"]
25
+ else: # Hard
26
+ source_lib = "requests"
27
+ target_lib = "httpx (Async)"
28
+ n_funcs = 6
29
+ template = "def {name}(url, data=None):\n import requests\n resp = requests.post(url, json=data)\n if resp.status_code == 200:\n return resp.text\n return None"
30
+ methods = ["upload", "sync_data", "backup", "telemetry", "notify", "patch_update"]
31
+
32
+ functions = []
33
+ for i in range(n_funcs):
34
+ name = methods[i] if i < len(methods) else f"func_{i}"
35
+ code = template.format(name=name, method=random.choice(["get", "post"]) if difficulty == "easy" else "")
36
+
37
+ functions.append({
38
+ "func_id": f"module_{difficulty}.py:{name}",
39
+ "code": code,
40
+ "library_functions_used": [f"{source_lib}.{m}" for m in (["get", "post"] if difficulty=="easy" else ["connect", "execute"])]
41
+ })
42
+
43
+ return {
44
+ "id": f"task_{difficulty}",
45
+ "difficulty": difficulty,
46
+ "source_lib": source_lib,
47
+ "target_lib": target_lib,
48
+ "functions": functions,
49
+ "max_steps": n_funcs + 2
50
+ }
51
+
52
+ TASKS = {
53
+ "task_easy": generate_migration_task("easy", seed=101),
54
+ "task_medium": generate_migration_task("medium", seed=102),
55
+ "task_hard": generate_migration_task("hard", seed=103)
56
+ }
uv.lock ADDED
The diff for this file is too large to render. See raw diff