Spaces:
Configuration error
Configuration error
Upload folder using huggingface_hub
Browse files- server/QUESTIONS.JSON +52 -0
- server/lint_coding_agent_environment.py +38 -28
server/QUESTIONS.JSON
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"1": {"lang": "Python", "task": "Fix Python 2 Print", "context": "print 'Hello World'", "ans": "print("},
|
| 3 |
+
"2": {"lang": "Python", "task": "Missing Colon in If", "context": "if True print('Success')", "ans": "True:"},
|
| 4 |
+
"3": {"lang": "Python", "task": "Indentation Error", "context": "def f():\nreturn 1", "ans": " return"},
|
| 5 |
+
"4": {"lang": "JavaScript", "task": "Const Reassignment", "context": "const x=10; x=20;", "ans": "let x"},
|
| 6 |
+
"5": {"lang": "JavaScript", "task": "Strict Equality", "context": "if (x == '5')", "ans": "==="},
|
| 7 |
+
"6": {"lang": "Python", "task": "Import Error", "context": "from math import floor_div", "ans": "floor"},
|
| 8 |
+
"7": {"lang": "Java", "task": "Missing Semicolon", "context": "int x = 5", "ans": "5;"},
|
| 9 |
+
"8": {"lang": "Java", "task": "String Comparison", "context": "if (str == 'admin')", "ans": ".equals("},
|
| 10 |
+
"9": {"lang": "Python", "task": "List Appending", "context": "my_list.add(5)", "ans": ".append("},
|
| 11 |
+
"10": {"lang": "Python", "task": "JSON Loads Logic", "context": "json.load('{\"a\":1}')", "ans": "json.loads"},
|
| 12 |
+
"11": {"lang": "C++", "task": "Pointer Syntax", "context": "int x = 5; int ptr = &x;", "ans": "int*"},
|
| 13 |
+
"12": {"lang": "Python", "task": "Mutable Default Arg", "context": "def f(a=[])", "ans": "None"},
|
| 14 |
+
"13": {"lang": "JavaScript", "task": "Template Literal", "context": "'Value: ' + x", "ans": "`${x}`"},
|
| 15 |
+
"14": {"lang": "Python", "task": "Type Error", "context": "len(555)", "ans": "str(555)"},
|
| 16 |
+
"15": {"lang": "Python", "task": "Dictionary Access", "context": "d.key", "ans": "d['key']"},
|
| 17 |
+
"16": {"lang": "C++", "task": "Namespace Missing", "context": "cout << 'Hi';", "ans": "std::cout"},
|
| 18 |
+
"17": {"lang": "Python", "task": "F-String Syntax", "context": "f'Val is {x'", "ans": "{x}"},
|
| 19 |
+
"18": {"lang": "JavaScript", "task": "Undefined Check", "context": "if (x = undefined)", "ans": "==="},
|
| 20 |
+
"19": {"lang": "Python", "task": "Except Block", "context": "except Error, e:", "ans": "as e:"},
|
| 21 |
+
"20": {"lang": "Rust", "task": "Ownership/Borrow", "context": "let s2 = s1;", "ans": ".clone()"},
|
| 22 |
+
"21": {"lang": "Python", "task": "Range Logic", "context": "range(10, 0)", "ans": "-1"},
|
| 23 |
+
"22": {"lang": "Java", "task": "Main Method", "context": "void main(String args)", "ans": "static"},
|
| 24 |
+
"23": {"lang": "Python", "task": "Open File Mode", "context": "open('f.txt', 'r').write()", "ans": "'w'"},
|
| 25 |
+
"24": {"lang": "JavaScript", "task": "Async Missing", "context": "await fetch(url)", "ans": "async"},
|
| 26 |
+
"25": {"lang": "Python", "task": "Class Init", "context": "def init(self):", "ans": "__init__"},
|
| 27 |
+
"26": {"lang": "C++", "task": "Vector Access", "context": "v.at[0]", "ans": "v[0]"},
|
| 28 |
+
"27": {"lang": "Python", "task": "Set Literal", "context": "s = {1, 1, 2}", "ans": "{1, 2}"},
|
| 29 |
+
"28": {"lang": "JavaScript", "task": "Arrow Function", "context": "func = () => { x }", "ans": "return"},
|
| 30 |
+
"29": {"lang": "Java", "task": "Boolean Value", "context": "boolean b = 1;", "ans": "true"},
|
| 31 |
+
"30": {"lang": "Python", "task": "Super Call", "context": "super.init()", "ans": "super().__init__"},
|
| 32 |
+
"31": {"lang": "Python", "task": "Global Keyword", "context": "x = 10; def f(): x = 5", "ans": "global x"},
|
| 33 |
+
"32": {"lang": "JavaScript", "task": "Map Return", "context": "arr.map(x => { x * 2 })", "ans": "return"},
|
| 34 |
+
"33": {"lang": "Python", "task": "Lambda Syntax", "context": "lambda x: return x", "ans": "x: x"},
|
| 35 |
+
"34": {"lang": "Java", "task": "Final Variable", "context": "final int x = 1; x = 2;", "ans": "int x"},
|
| 36 |
+
"35": {"lang": "Python", "task": "Init Self Missing", "context": "def __init__():", "ans": "(self)"},
|
| 37 |
+
"36": {"lang": "C++", "task": "Memory Leak", "context": "int* p = new int[5];", "ans": "delete[]"},
|
| 38 |
+
"37": {"lang": "Python", "task": "Zip Unpack", "context": "for a, b in zip(l1)", "ans": "l1, l2"},
|
| 39 |
+
"38": {"lang": "JavaScript", "task": "Nullish Coalescing", "context": "x || 'default'", "ans": "??"},
|
| 40 |
+
"39": {"lang": "Python", "task": "Format Specifier", "context": "'%.2f' % 'str'", "ans": "1.23"},
|
| 41 |
+
"40": {"lang": "Java", "task": "Inheritance", "context": "class A implements B", "ans": "extends"},
|
| 42 |
+
"41": {"lang": "Python", "task": "Docstring", "context": "'This is doc' def f():", "ans": "def f():\n \"\"\""},
|
| 43 |
+
"42": {"lang": "JavaScript", "task": "Promise Resolve", "context": "new Promise((res) => res)", "ans": "res()"},
|
| 44 |
+
"43": {"lang": "Python", "task": "Sort Key", "context": "list.sort(key=None)", "ans": "lambda"},
|
| 45 |
+
"44": {"lang": "C++", "task": "For Loop Iterator", "context": "for(i=0; i<5; i++)", "ans": "int i"},
|
| 46 |
+
"45": {"lang": "Python", "task": "Regex Escape", "context": "re.search('\\d', s)", "ans": "r'\\d'"},
|
| 47 |
+
"46": {"lang": "Java", "task": "ArrayList Type", "context": "List<int> l;", "ans": "Integer"},
|
| 48 |
+
"47": {"lang": "Python", "task": "Static Method", "context": "def f(self):", "ans": "@staticmethod"},
|
| 49 |
+
"48": {"lang": "JavaScript", "task": "Module Export", "context": "export f(){}", "ans": "export function"},
|
| 50 |
+
"49": {"lang": "Python", "task": "Async Loop", "context": "for x in async_gen():", "ans": "async for"},
|
| 51 |
+
"50": {"lang": "Python", "task": "Final Challenge", "context": "eval(input())", "ans": "ast.literal_eval"}
|
| 52 |
+
}
|
server/lint_coding_agent_environment.py
CHANGED
|
@@ -11,6 +11,8 @@ A simple test environment that echoes back messages sent to it.
|
|
| 11 |
Perfect for testing HTTP server infrastructure.
|
| 12 |
"""
|
| 13 |
|
|
|
|
|
|
|
| 14 |
from uuid import uuid4
|
| 15 |
from openenv.core.env_server.interfaces import Environment
|
| 16 |
from openenv.core.env_server.types import State
|
|
@@ -20,68 +22,76 @@ try:
|
|
| 20 |
except ImportError:
|
| 21 |
from models import LintCodingAgentAction, LintCodingAgentObservation
|
| 22 |
|
| 23 |
-
# MANDATORY: Real-world task data for Level 1, 10, and 20 (Easy, Medium, Hard)
|
| 24 |
-
CURRICULUM = {
|
| 25 |
-
1: {"lang": "Python", "task": "Fix Syntax: Python 2 to 3", "context": "print 'Hello'", "ans": "print("},
|
| 26 |
-
10: {"lang": "Python", "task": "Libraries: Fix JSON load", "context": "json.load('{\"a\":1}')", "ans": "json.loads"},
|
| 27 |
-
20: {"lang": "Python", "task": "Optimization: Use Set lookup", "context": "if x in [1,2,3]:", "ans": "set(["},
|
| 28 |
-
}
|
| 29 |
-
|
| 30 |
class LintCodingAgentEnvironment(Environment):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
SUPPORTS_CONCURRENT_SESSIONS: bool = True
|
| 32 |
|
| 33 |
def __init__(self):
|
| 34 |
# Initialize internal state using OpenEnv's State object
|
| 35 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 36 |
self.level = 1
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
def reset(self) -> LintCodingAgentObservation:
|
| 40 |
-
"""Reset the environment to the
|
| 41 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 42 |
self.level = 1
|
| 43 |
-
return self._get_observation("
|
| 44 |
|
| 45 |
def step(self, action: LintCodingAgentAction) -> LintCodingAgentObservation:
|
| 46 |
-
"""Process the agent's
|
| 47 |
self._state.step_count += 1
|
| 48 |
|
| 49 |
-
#
|
| 50 |
-
|
| 51 |
|
| 52 |
-
#
|
| 53 |
-
# Accessing 'code_solution' and 'explanation' from your strict Action model
|
| 54 |
solution = action.code_solution
|
| 55 |
explanation = action.explanation
|
| 56 |
|
| 57 |
-
# Case-insensitive
|
| 58 |
-
is_correct =
|
| 59 |
|
| 60 |
-
# 3. REWARD LOGIC
|
| 61 |
if is_correct:
|
| 62 |
reward = 1.0
|
| 63 |
self.level += 1
|
| 64 |
-
feedback = f"Correct! Level {self.level - 1} cleared.
|
| 65 |
else:
|
| 66 |
reward = -0.1
|
| 67 |
-
feedback = f"Incorrect. Retry Level {self.level}. Hint: {
|
| 68 |
|
| 69 |
-
#
|
| 70 |
-
# Finish if max levels hit or if the agent explicitly quits
|
| 71 |
done = self.level > self.max_levels or "QUIT" in solution.upper()
|
| 72 |
|
| 73 |
return self._get_observation(feedback, reward, done)
|
| 74 |
|
| 75 |
def _get_observation(self, feedback: str, reward: float, done: bool) -> LintCodingAgentObservation:
|
| 76 |
-
"""Helper to package internal
|
| 77 |
-
|
|
|
|
| 78 |
|
| 79 |
-
# This MUST match the fields in your LintCodingAgentObservation exactly
|
| 80 |
return LintCodingAgentObservation(
|
| 81 |
level=self.level,
|
| 82 |
-
language=
|
| 83 |
-
problem_statement=f"{feedback} |
|
| 84 |
-
code_context=
|
| 85 |
last_test_results=feedback,
|
| 86 |
reward=reward,
|
| 87 |
done=done,
|
|
|
|
| 11 |
Perfect for testing HTTP server infrastructure.
|
| 12 |
"""
|
| 13 |
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
from uuid import uuid4
|
| 17 |
from openenv.core.env_server.interfaces import Environment
|
| 18 |
from openenv.core.env_server.types import State
|
|
|
|
| 22 |
except ImportError:
|
| 23 |
from models import LintCodingAgentAction, LintCodingAgentObservation
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
class LintCodingAgentEnvironment(Environment):
|
| 26 |
+
"""
|
| 27 |
+
A data-driven environment that loads a 50-level curriculum from a JSON file.
|
| 28 |
+
Designed for professional code-linting agent evaluation.
|
| 29 |
+
"""
|
| 30 |
SUPPORTS_CONCURRENT_SESSIONS: bool = True
|
| 31 |
|
| 32 |
def __init__(self):
|
| 33 |
# Initialize internal state using OpenEnv's State object
|
| 34 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 35 |
self.level = 1
|
| 36 |
+
|
| 37 |
+
# --- DATA LOADING BLOCK ---
|
| 38 |
+
# Get the path to QUESTIONS.json relative to this file to avoid pathing errors in Docker
|
| 39 |
+
current_dir = os.path.dirname(os.path.abspath(__file__))
|
| 40 |
+
json_path = os.path.join(current_dir, "QUESTIONS.json")
|
| 41 |
+
|
| 42 |
+
try:
|
| 43 |
+
with open(json_path, "r") as f:
|
| 44 |
+
self.curriculum = json.load(f)
|
| 45 |
+
except Exception as e:
|
| 46 |
+
print(f"CRITICAL ERROR: Could not load QUESTIONS.json: {e}")
|
| 47 |
+
# Minimal fallback to prevent total server crash
|
| 48 |
+
self.curriculum = {"1": {"lang": "Python", "task": "Fallback", "context": "print 'Hi'", "ans": "print("}}
|
| 49 |
+
|
| 50 |
+
self.max_levels = len(self.curriculum)
|
| 51 |
|
| 52 |
def reset(self) -> LintCodingAgentObservation:
|
| 53 |
+
"""Reset the environment to the beginning of the curriculum."""
|
| 54 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 55 |
self.level = 1
|
| 56 |
+
return self._get_observation("Curriculum Loaded. Ready for Level 1.", 0.0, False)
|
| 57 |
|
| 58 |
def step(self, action: LintCodingAgentAction) -> LintCodingAgentObservation:
|
| 59 |
+
"""Process the agent's solution against the current JSON task."""
|
| 60 |
self._state.step_count += 1
|
| 61 |
|
| 62 |
+
# Get current level data (JSON keys are strings)
|
| 63 |
+
task_data = self.curriculum.get(str(self.level), self.curriculum.get("1"))
|
| 64 |
|
| 65 |
+
# Logic: Extract fields from the strict Action model
|
|
|
|
| 66 |
solution = action.code_solution
|
| 67 |
explanation = action.explanation
|
| 68 |
|
| 69 |
+
# Case-insensitive grading logic
|
| 70 |
+
is_correct = task_data["ans"].lower() in solution.lower()
|
| 71 |
|
|
|
|
| 72 |
if is_correct:
|
| 73 |
reward = 1.0
|
| 74 |
self.level += 1
|
| 75 |
+
feedback = f"Correct! Level {self.level - 1} cleared. Reason: {explanation}"
|
| 76 |
else:
|
| 77 |
reward = -0.1
|
| 78 |
+
feedback = f"Incorrect fix. Retry Level {self.level}. Hint: {task_data['task']}"
|
| 79 |
|
| 80 |
+
# Done if all levels complete or agent gives up
|
|
|
|
| 81 |
done = self.level > self.max_levels or "QUIT" in solution.upper()
|
| 82 |
|
| 83 |
return self._get_observation(feedback, reward, done)
|
| 84 |
|
| 85 |
def _get_observation(self, feedback: str, reward: float, done: bool) -> LintCodingAgentObservation:
|
| 86 |
+
"""Helper to package internal data into the strict Observation Pydantic model."""
|
| 87 |
+
# Use string indexing for JSON curriculum access
|
| 88 |
+
task_data = self.curriculum.get(str(self.level), self.curriculum.get("1"))
|
| 89 |
|
|
|
|
| 90 |
return LintCodingAgentObservation(
|
| 91 |
level=self.level,
|
| 92 |
+
language=task_data["lang"],
|
| 93 |
+
problem_statement=f"{feedback} | Goal: {task_data['task']}",
|
| 94 |
+
code_context=task_data["context"],
|
| 95 |
last_test_results=feedback,
|
| 96 |
reward=reward,
|
| 97 |
done=done,
|