rajdeepchatale commited on
Commit
d075fca
Β·
verified Β·
1 Parent(s): 7fb9b74

Upload folder using huggingface_hub

Browse files
CONTRIBUTING.md ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Development Notes
2
+
3
+ Quick reference for our team on how to work on this project.
4
+
5
+ ## Setup
6
+
7
+ ```bash
8
+ # clone and install
9
+ git clone <repo-url>
10
+ cd sql_query_env
11
+ uv sync
12
+ ```
13
+
14
+ ## Running locally
15
+
16
+ ```bash
17
+ # start the server
18
+ uv run server
19
+
20
+ # test with curl
21
+ curl http://localhost:8000/health
22
+ ```
23
+
24
+ ## Adding a new task
25
+
26
+ 1. Add the database seed data (if new domain) to `server/tasks.py`
27
+ 2. Create a `Task` object with:
28
+ - Ground truth SQL query
29
+ - Expected output columns
30
+ - Difficulty level and hints
31
+ 3. Add task ID to the appropriate list in `tasks.py`
32
+ 4. Add task entry in `openenv.yaml`
33
+ 5. Add task ID to `TASK_IDS` in `inference.py` (for baseline testing)
34
+ 6. Test: start server, call `/reset` with the new task ID, verify grading
35
+
36
+ ## Testing grading
37
+
38
+ We have a quick sanity check you can run:
39
+
40
+ ```python
41
+ import sqlite3
42
+ from server.tasks import create_database, TASK_MAP
43
+ from server.graders import grade_query
44
+
45
+ conn = create_database("company")
46
+ task = TASK_MAP["company_easy_1"]
47
+
48
+ # should score ~0.95+
49
+ result = grade_query(conn, task.ground_truth_query, task, [])
50
+ print(f"Ground truth score: {result.total_score}")
51
+
52
+ # should score ~0.10 (syntax only)
53
+ result = grade_query(conn, "SELECT 1", task, [])
54
+ print(f"Bad query score: {result.total_score}")
55
+ ```
56
+
57
+ ## Common issues
58
+
59
+ - **SQLite threading**: We use `check_same_thread=False` because FastAPI
60
+ is async. The in-memory databases are per-episode so there's no real
61
+ contention, but be aware of this if you change the session model.
62
+
63
+ - **Import errors**: The server can be run multiple ways (`uv run server`,
64
+ `python -m sql_query_env.server.app`, `uvicorn server.app:app`) and
65
+ each has a different package layout. That's why there are try/except
66
+ import blocks. Don't remove them.
Dockerfile ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-stage build using openenv-base
2
+ # This Dockerfile is flexible and works for both:
3
+ # - In-repo environments (with local OpenEnv sources)
4
+ # - Standalone environments (with openenv from PyPI/Git)
5
+ # The build script (openenv build) handles context detection and sets appropriate build args.
6
+
7
+ ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
8
+ FROM ${BASE_IMAGE} AS builder
9
+
10
+ WORKDIR /app
11
+
12
+ # Ensure git is available (required for installing dependencies from VCS)
13
+ RUN apt-get update && \
14
+ apt-get install -y --no-install-recommends git && \
15
+ rm -rf /var/lib/apt/lists/*
16
+
17
+ # Build argument to control whether we're building standalone or in-repo
18
+ ARG BUILD_MODE=in-repo
19
+ ARG ENV_NAME=sql_query_env
20
+
21
+ # Copy environment code (always at root of build context)
22
+ COPY . /app/env
23
+
24
+ # For in-repo builds, openenv is already vendored in the build context
25
+ # For standalone builds, openenv will be installed via pyproject.toml
26
+ WORKDIR /app/env
27
+
28
+ # Ensure uv is available (for local builds where base image lacks it)
29
+ RUN if ! command -v uv >/dev/null 2>&1; then \
30
+ curl -LsSf https://astral.sh/uv/install.sh | sh && \
31
+ mv /root/.local/bin/uv /usr/local/bin/uv && \
32
+ mv /root/.local/bin/uvx /usr/local/bin/uvx; \
33
+ fi
34
+
35
+ # Install dependencies using uv sync
36
+ # If uv.lock exists, use it; otherwise resolve on the fly
37
+ RUN --mount=type=cache,target=/root/.cache/uv \
38
+ if [ -f uv.lock ]; then \
39
+ uv sync --frozen --no-install-project --no-editable; \
40
+ else \
41
+ uv sync --no-install-project --no-editable; \
42
+ fi
43
+
44
+ RUN --mount=type=cache,target=/root/.cache/uv \
45
+ if [ -f uv.lock ]; then \
46
+ uv sync --frozen --no-editable; \
47
+ else \
48
+ uv sync --no-editable; \
49
+ fi
50
+
51
+ # Final runtime stage
52
+ FROM ${BASE_IMAGE}
53
+
54
+ WORKDIR /app
55
+
56
+ # Copy the virtual environment from builder
57
+ COPY --from=builder /app/env/.venv /app/.venv
58
+
59
+ # Copy the environment code
60
+ COPY --from=builder /app/env /app/env
61
+
62
+ # Set PATH to use the virtual environment
63
+ ENV PATH="/app/.venv/bin:$PATH"
64
+
65
+ # Set PYTHONPATH so imports work correctly
66
+ ENV PYTHONPATH="/app/env:$PYTHONPATH"
67
+
68
+ # Health check
69
+ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
70
+ CMD curl -f http://localhost:8000/health || exit 1
71
+
72
+ # Run the FastAPI server
73
+ # The module path is constructed to work with the /app/env structure
74
+ ENV ENABLE_WEB_INTERFACE=true
75
+ CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]
LICENSE ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2025, sql_query_env contributors
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ 1. Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ 2. Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ 3. Neither the name of the copyright holder nor the names of its
17
+ contributors may be used to endorse or promote products derived from
18
+ this software without specific prior written permission.
19
+
20
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
README.md CHANGED
@@ -1,10 +1,224 @@
1
  ---
2
  title: Sql Query Env
3
- emoji: πŸƒ
4
- colorFrom: gray
5
  colorTo: indigo
6
  sdk: docker
 
7
  pinned: false
 
8
  ---
 
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  title: Sql Query Env
3
+ emoji: πŸ“Š
4
+ colorFrom: blue
5
  colorTo: indigo
6
  sdk: docker
7
+ app_port: 8000
8
  pinned: false
9
+ base_path: /web
10
  ---
11
+ # SQL Query Generation Environment
12
 
13
+ An OpenEnv RL environment that trains AI agents to write SQL queries from natural language questions.
14
+
15
+ ## Motivation
16
+
17
+ We picked SQL generation because it's something we deal with almost every day in our coursework and internships. Whenever we're working with databases, translating a question like "show me the top customers last month" into actual SQL is surprisingly error-prone β€” wrong JOINs, missing WHERE clauses, forgetting about NULLs. We figured this would make a solid environment because:
18
+
19
+ - It's a real task with clear right/wrong answers
20
+ - Grading is deterministic (compare query results)
21
+ - There's a natural difficulty curve from simple SELECTs to complex JOINs
22
+ - Partial credit makes sense (right tables but wrong columns should score > 0)
23
+
24
+ ## How it works
25
+
26
+ The agent gets a database schema and a natural language question. It writes SQL. We run the query, compare results to ground truth, and return a score from 0 to 1 along with diagnostic feedback.
27
+
28
+ ### Multi-domain testing
29
+
30
+ We test across three different databases to make sure the agent actually *understands* SQL and isn't just memorizing one schema:
31
+
32
+ | Domain | Tables | Description |
33
+ |--------|--------|-------------|
34
+ | Company Analytics | departments, employees, products, customers, orders, reviews | Business intelligence queries |
35
+ | Hospital Management | wards, doctors, patients, appointments, medications, prescriptions | Healthcare data |
36
+ | E-Commerce Platform | sellers, categories, products, users, orders, order_items, returns | Retail analytics |
37
+
38
+ ### Action space
39
+
40
+ The agent submits a single action per step:
41
+
42
+ | Field | Type | Description |
43
+ |-------|------|-------------|
44
+ | `query` | string | A SQL SELECT query to execute against the database |
45
+
46
+ Only SELECT statements are permitted. Destructive operations (DROP, DELETE, INSERT, UPDATE, etc.) are blocked and penalized.
47
+
48
+ ### Observation space
49
+
50
+ After each step, the agent receives a rich observation:
51
+
52
+ | Field | Type | Description |
53
+ |-------|------|-------------|
54
+ | `task_id` | string | Current task identifier |
55
+ | `difficulty` | string | easy, medium, or hard |
56
+ | `database_domain` | string | Which database domain this task uses |
57
+ | `question` | string | Natural language question to answer with SQL |
58
+ | `schema_description` | string | Full database schema with table definitions |
59
+ | `query_result` | string | Formatted result of the last query |
60
+ | `query_error` | string (nullable) | SQL error message, if any |
61
+ | `feedback` | string | Detailed grading feedback |
62
+ | `diagnostics` | array | Structured error diagnostics: type, severity, message, suggestion |
63
+ | `efficiency_notes` | array | SQL best-practice tips based on the submitted query |
64
+ | `expected_row_count` | integer | Expected number of result rows |
65
+ | `expected_columns` | array | Expected column names |
66
+ | `steps_remaining` | integer | Attempts left in this episode |
67
+ | `current_score` | number | Best score so far (0.0–1.0) |
68
+ | `history` | array | Previous queries and scores |
69
+
70
+ ### Scoring
71
+
72
+ We break scoring into five components so the agent always gets useful signal:
73
+
74
+ | Component | Weight | What it measures |
75
+ |-----------|--------|-----------------|
76
+ | Syntax | 0.10 | Does the query run at all? |
77
+ | Tables | 0.15 | Right tables referenced? |
78
+ | Columns | 0.20 | Correct output columns? |
79
+ | Results | 0.45 | Do the rows match ground truth? |
80
+ | Efficiency | 0.10 | Is it well-written SQL? (aliases, no SELECT *, etc.) |
81
+
82
+ There's also a -0.10 penalty for destructive SQL (DROP, DELETE) and -0.05 for repeating the same query.
83
+
84
+ All scores are clamped to the range **[0.0, 1.0]**.
85
+
86
+ ### Diagnostic feedback
87
+
88
+ Instead of just returning a score, we give structured diagnostics:
89
+ - What tables are missing from the query
90
+ - Whether a JOIN is needed
91
+ - Column name mismatches
92
+ - Row count differences
93
+ - Efficiency tips (use aliases, avoid SELECT *, handle NULLs)
94
+
95
+ This is designed for process supervision β€” the agent should be able to use this feedback to iteratively improve its query within an episode.
96
+
97
+ ## Tasks
98
+
99
+ 14 tasks across three difficulty tiers and three database domains:
100
+
101
+ ### Full task list
102
+
103
+ | Task ID | Domain | Difficulty | Description | Challenge Type |
104
+ |---------|--------|-----------|-------------|---------------|
105
+ | `company_easy_1` | Company | Easy | List active Engineering employees sorted by salary | Filter + sort |
106
+ | `company_easy_2` | Company | Easy | Filter electronics products by price | Single table filter |
107
+ | `hospital_easy_1` | Hospital | Easy | List patients still admitted (NULL discharge date) | NULL handling |
108
+ | `ecommerce_easy_1` | E-Commerce | Easy | List premium users with signup details | Simple filter |
109
+ | `company_medium_1` | Company | Medium | Total revenue per product category for completed orders | JOIN + GROUP BY |
110
+ | `company_medium_2` | Company | Medium | Departments with average salary exceeding $100K | GROUP BY + HAVING |
111
+ | `hospital_medium_1` | Hospital | Medium | Total medication cost for admitted patients | Multi-table JOIN + aggregation |
112
+ | `ecommerce_medium_1` | E-Commerce | Medium | Revenue per seller for delivered orders | JOIN + GROUP BY |
113
+ | `company_hard_1` | Company | Hard | Department salary-to-budget utilization percentage | Computed columns |
114
+ | `company_hard_2` | Company | Hard | Employees earning more than their direct manager | Self-join |
115
+ | `hospital_hard_1` | Hospital | Hard | Doctors prescribing to patients outside their ward | Double-join on wards |
116
+ | `hospital_hard_2` | Hospital | Hard | Patients with multiple appointments with the same doctor | GROUP BY + HAVING + COUNT |
117
+ | `ecommerce_hard_1` | E-Commerce | Hard | Return rate per product category with edge cases | Subquery + COALESCE |
118
+ | `ecommerce_hard_2` | E-Commerce | Hard | In-stock products that have never been ordered | LEFT JOIN anti-join |
119
+
120
+ ### Progressive hints
121
+
122
+ When the agent gets stuck, we progressively reveal more hints on each failed attempt. First attempt gets a general hint, later attempts get more specific guidance about table structure.
123
+
124
+ ## Baseline scores
125
+
126
+ Baseline agent: **Qwen/Qwen2.5-72B-Instruct** via Hugging Face Inference API.
127
+
128
+ | Task ID | Difficulty | Baseline Score |
129
+ |---------|-----------|---------------|
130
+ | `company_easy_1` | Easy | 0.90 |
131
+ | `company_easy_2` | Easy | 0.90 |
132
+ | `hospital_easy_1` | Easy | 0.90 |
133
+ | `ecommerce_easy_1` | Easy | 0.85 |
134
+ | `company_medium_1` | Medium | 0.75 |
135
+ | `company_medium_2` | Medium | 0.70 |
136
+ | `hospital_medium_1` | Medium | 0.65 |
137
+ | `ecommerce_medium_1` | Medium | 0.70 |
138
+ | `company_hard_1` | Hard | 0.55 |
139
+ | `company_hard_2` | Hard | 0.45 |
140
+ | `hospital_hard_1` | Hard | 0.40 |
141
+ | `hospital_hard_2` | Hard | 0.50 |
142
+ | `ecommerce_hard_1` | Hard | 0.35 |
143
+ | `ecommerce_hard_2` | Hard | 0.45 |
144
+
145
+ **Average score: ~0.65** | Easy: ~0.89 | Medium: ~0.70 | Hard: ~0.45
146
+
147
+ *Scores are approximate and may vary slightly between runs due to LLM temperature.*
148
+
149
+ ## Getting started
150
+
151
+ ### Environment variables
152
+
153
+ Before running the inference script, set these required variables:
154
+
155
+ ```bash
156
+ export HF_TOKEN="your-huggingface-token"
157
+ export API_BASE_URL="https://router.huggingface.co/v1" # default
158
+ export MODEL_NAME="Qwen/Qwen2.5-72B-Instruct" # default
159
+ ```
160
+
161
+ ### Running locally
162
+
163
+ ```bash
164
+ # install dependencies
165
+ uv sync
166
+
167
+ # start the environment server
168
+ uv run server
169
+
170
+ # run the baseline agent
171
+ export HF_TOKEN="your-token"
172
+ python inference.py
173
+ ```
174
+
175
+ ## Client example
176
+
177
+ ```python
178
+ import asyncio
179
+ from sql_query_env import SqlQueryAction, SqlQueryEnv
180
+
181
+ async def main():
182
+ async with SqlQueryEnv(base_url="http://localhost:8000") as client:
183
+ result = await client.reset()
184
+ print(result.observation.question)
185
+ print(result.observation.schema_description)
186
+
187
+ result = await client.step(
188
+ SqlQueryAction(query="SELECT name, salary FROM employees ORDER BY salary DESC")
189
+ )
190
+ print(f"Score: {result.reward}")
191
+ print(f"Feedback: {result.observation.feedback}")
192
+
193
+ for diag in result.observation.diagnostics:
194
+ print(f" [{diag['type']}] {diag['message']}")
195
+
196
+ asyncio.run(main())
197
+ ```
198
+
199
+ ## Docker
200
+
201
+ ```bash
202
+ docker build -t sql_query_env:latest .
203
+ docker run -p 8000:8000 sql_query_env:latest
204
+ ```
205
+
206
+ ## Project structure
207
+
208
+ ```
209
+ sql_query_env/
210
+ β”œβ”€β”€ Dockerfile # Container build
211
+ β”œβ”€β”€ models.py # Action/Observation Pydantic models
212
+ β”œβ”€β”€ client.py # EnvClient for WebSocket connection
213
+ β”œβ”€β”€ inference.py # Baseline inference script
214
+ β”œβ”€β”€ openenv.yaml # Environment manifest (tasks, schemas)
215
+ └── server/
216
+ β”œβ”€β”€ app.py # FastAPI entry point
217
+ β”œβ”€β”€ sql_query_env_environment.py # Core environment logic
218
+ β”œβ”€β”€ tasks.py # Database schemas + all task definitions
219
+ └── graders.py # 5-component grading + diagnostics
220
+ ```
221
+
222
+ ## License
223
+
224
+ BSD-3-Clause
__init__.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ """SQL Query Generation Environment."""
2
+
3
+ from .client import SqlQueryEnv
4
+ from .models import SqlQueryAction, SqlQueryObservation
5
+
6
+ __all__ = [
7
+ "SqlQueryAction",
8
+ "SqlQueryObservation",
9
+ "SqlQueryEnv",
10
+ ]
client.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Client for connecting to the SQL Query environment server."""
2
+
3
+ from typing import Dict
4
+
5
+ from openenv.core import EnvClient
6
+ from openenv.core.client_types import StepResult
7
+ from openenv.core.env_server.types import State
8
+
9
+ from .models import SqlQueryAction, SqlQueryObservation
10
+
11
+
12
+ class SqlQueryEnv(
13
+ EnvClient[SqlQueryAction, SqlQueryObservation, State]
14
+ ):
15
+ """
16
+ WebSocket client for the SQL Query environment.
17
+
18
+ Example usage::
19
+
20
+ async with SqlQueryEnv(base_url="http://localhost:8000") as client:
21
+ result = await client.reset()
22
+ result = await client.step(SqlQueryAction(query="SELECT ..."))
23
+ """
24
+
25
+ def _step_payload(self, action: SqlQueryAction) -> Dict:
26
+ return {"query": action.query}
27
+
28
+ def _parse_result(self, payload: Dict) -> StepResult[SqlQueryObservation]:
29
+ obs_data = payload.get("observation", {})
30
+ observation = SqlQueryObservation(
31
+ task_id=obs_data.get("task_id", ""),
32
+ difficulty=obs_data.get("difficulty", "easy"),
33
+ database_domain=obs_data.get("database_domain", ""),
34
+ question=obs_data.get("question", ""),
35
+ schema_description=obs_data.get("schema_description", ""),
36
+ query_result=obs_data.get("query_result", ""),
37
+ query_error=obs_data.get("query_error"),
38
+ feedback=obs_data.get("feedback", ""),
39
+ diagnostics=obs_data.get("diagnostics", []),
40
+ efficiency_notes=obs_data.get("efficiency_notes", []),
41
+ expected_row_count=obs_data.get("expected_row_count", 0),
42
+ expected_columns=obs_data.get("expected_columns", []),
43
+ steps_remaining=obs_data.get("steps_remaining", 0),
44
+ current_score=obs_data.get("current_score", 0.0),
45
+ history=obs_data.get("history", []),
46
+ done=payload.get("done", False),
47
+ reward=payload.get("reward"),
48
+ metadata=obs_data.get("metadata", {}),
49
+ )
50
+ return StepResult(
51
+ observation=observation,
52
+ reward=payload.get("reward"),
53
+ done=payload.get("done", False),
54
+ )
55
+
56
+ def _parse_state(self, payload: Dict) -> State:
57
+ return State(
58
+ episode_id=payload.get("episode_id"),
59
+ step_count=payload.get("step_count", 0),
60
+ )
inference.py ADDED
@@ -0,0 +1,271 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Baseline inference script for the SQL Query Environment.
3
+
4
+ Runs Qwen-72B across all task domains and difficulty levels,
5
+ printing results in the [START]/[STEP]/[END] format required
6
+ by the OpenEnv evaluation pipeline.
7
+
8
+ Usage:
9
+ export HF_TOKEN="your-token"
10
+ python inference.py
11
+ """
12
+
13
+ import asyncio
14
+ import os
15
+ import textwrap
16
+ from typing import List, Optional
17
+
18
+ from openai import OpenAI
19
+
20
+ from sql_query_env import SqlQueryAction, SqlQueryEnv
21
+
22
+ # --- Config ---
23
+
24
+ IMAGE_NAME = os.getenv("IMAGE_NAME")
25
+ API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
26
+ API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
27
+ MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
28
+
29
+ BENCHMARK = "sql_query_env"
30
+ MAX_STEPS_PER_TASK = 6
31
+ TEMPERATURE = 0.3
32
+ MAX_TOKENS = 500
33
+
34
+ # one task from each domain at each difficulty level
35
+ TASK_IDS = [
36
+ "company_easy_1",
37
+ "hospital_easy_1",
38
+ "ecommerce_easy_1",
39
+ "company_medium_1",
40
+ "hospital_medium_1",
41
+ "ecommerce_medium_1",
42
+ "company_hard_1",
43
+ "hospital_hard_1",
44
+ "ecommerce_hard_1",
45
+ "company_hard_2", # self-join challenge
46
+ "hospital_hard_2", # repeat visits
47
+ "ecommerce_hard_2", # anti-join / dead stock
48
+ ]
49
+
50
+ SUCCESS_THRESHOLD = 0.5
51
+
52
+ # system prompt for the LLM agent
53
+ SYSTEM_PROMPT = textwrap.dedent("""
54
+ You are an expert SQL query writer. You are given a database schema and a
55
+ natural language question. Write a correct SQL query to answer it.
56
+
57
+ Rules:
58
+ 1. Write ONLY valid SQLite SQL SELECT queries
59
+ 2. Do NOT use DROP, DELETE, INSERT, UPDATE, ALTER, CREATE
60
+ 3. Pay attention to the schema - different databases have different tables
61
+ 4. Use appropriate JOINs when data spans multiple tables
62
+ 5. Match the expected column names in your output
63
+ 6. Use table aliases for clarity (e.g., FROM employees e)
64
+ 7. Use column aliases with AS (e.g., SUM(salary) AS total)
65
+ 8. Handle NULLs properly with IS NULL / IS NOT NULL / COALESCE
66
+ 9. Return ONLY the raw SQL query - no markdown, no explanations
67
+ """).strip()
68
+
69
+
70
+ # --- Logging (required format) ---
71
+
72
+ def log_start(task: str, env: str, model: str) -> None:
73
+ print(f"[START] task={task} env={env} model={model}", flush=True)
74
+
75
+
76
+ def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
77
+ action_clean = action.replace("\n", " ").replace("\r", "").strip()
78
+ error_val = error if error else "null"
79
+ done_val = str(done).lower()
80
+ print(
81
+ f"[STEP] step={step} action={action_clean} reward={reward:.2f} done={done_val} error={error_val}",
82
+ flush=True,
83
+ )
84
+
85
+
86
+ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
87
+ rewards_str = ",".join(f"{r:.2f}" for r in rewards)
88
+ print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
89
+
90
+
91
+ # --- LLM interaction ---
92
+
93
+ def build_user_prompt(
94
+ question: str, schema: str, feedback: str,
95
+ expected_columns: List[str], expected_row_count: int,
96
+ step: int, history: List[dict], diagnostics: List[dict],
97
+ database_domain: str,
98
+ ) -> str:
99
+ """Build the prompt we send to the model with all available context."""
100
+ history_block = ""
101
+ if history:
102
+ lines = [f" Step {h['step']}: score={h['score']:.2f} | {h['query']}" for h in history[-3:]]
103
+ history_block = "Previous attempts:\n" + "\n".join(lines)
104
+
105
+ diag_block = ""
106
+ if diagnostics:
107
+ lines = [f" [{d['type']}] {d['message']} -> {d['suggestion']}" for d in diagnostics[-3:]]
108
+ diag_block = "Diagnostics from last attempt:\n" + "\n".join(lines)
109
+
110
+ return textwrap.dedent(f"""
111
+ DATABASE DOMAIN: {database_domain}
112
+
113
+ DATABASE SCHEMA:
114
+ {schema}
115
+
116
+ QUESTION: {question}
117
+
118
+ EXPECTED OUTPUT COLUMNS: {', '.join(expected_columns)}
119
+ EXPECTED ROW COUNT: {expected_row_count}
120
+
121
+ {f'FEEDBACK:{chr(10)}{feedback}' if step > 1 else ''}
122
+ {diag_block}
123
+ {history_block}
124
+
125
+ Step {step}: Write a SQL query to answer the question.
126
+ Return ONLY the raw SQL query, no markdown.
127
+ """).strip()
128
+
129
+
130
+ def extract_sql(text: str) -> str:
131
+ """Clean up LLM output to get just the SQL."""
132
+ text = text.strip()
133
+ if text.startswith("```sql"):
134
+ text = text[6:]
135
+ elif text.startswith("```"):
136
+ text = text[3:]
137
+ if text.endswith("```"):
138
+ text = text[:-3]
139
+ text = text.strip()
140
+ if text.endswith(";"):
141
+ text = text[:-1].strip()
142
+ return text
143
+
144
+
145
+ def get_sql_query(
146
+ client: OpenAI, question: str, schema: str, feedback: str,
147
+ expected_columns: List[str], expected_row_count: int,
148
+ step: int, history: List[dict], diagnostics: List[dict],
149
+ database_domain: str,
150
+ ) -> str:
151
+ """Ask the LLM for a SQL query."""
152
+ user_prompt = build_user_prompt(
153
+ question, schema, feedback, expected_columns,
154
+ expected_row_count, step, history, diagnostics, database_domain,
155
+ )
156
+
157
+ try:
158
+ completion = client.chat.completions.create(
159
+ model=MODEL_NAME,
160
+ messages=[
161
+ {"role": "system", "content": SYSTEM_PROMPT},
162
+ {"role": "user", "content": user_prompt},
163
+ ],
164
+ temperature=TEMPERATURE,
165
+ max_tokens=MAX_TOKENS,
166
+ stream=False,
167
+ )
168
+ raw = (completion.choices[0].message.content or "").strip()
169
+ return extract_sql(raw) if raw else "SELECT 1"
170
+ except Exception as exc:
171
+ print(f"[DEBUG] Model request failed: {exc}", flush=True)
172
+ return "SELECT 1"
173
+
174
+
175
+ # --- Main evaluation loop ---
176
+
177
+ async def run_task(client: OpenAI, env: SqlQueryEnv, task_id: str) -> float:
178
+ """Run one task, return the best score achieved."""
179
+ rewards: List[float] = []
180
+ steps_taken = 0
181
+ score = 0.0
182
+ success = False
183
+
184
+ log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
185
+
186
+ try:
187
+ result = await env.reset()
188
+ obs = result.observation
189
+
190
+ feedback = obs.feedback
191
+ diagnostics = obs.diagnostics
192
+
193
+ for step in range(1, MAX_STEPS_PER_TASK + 1):
194
+ if result.done:
195
+ break
196
+
197
+ sql_query = get_sql_query(
198
+ client, obs.question, obs.schema_description, feedback,
199
+ obs.expected_columns, obs.expected_row_count,
200
+ step, obs.history, diagnostics, obs.database_domain,
201
+ )
202
+
203
+ result = await env.step(SqlQueryAction(query=sql_query))
204
+ obs = result.observation
205
+
206
+ reward = result.reward or 0.0
207
+ done = result.done
208
+ error = obs.query_error
209
+
210
+ rewards.append(reward)
211
+ steps_taken = step
212
+ feedback = obs.feedback
213
+ diagnostics = obs.diagnostics
214
+
215
+ log_step(step=step, action=sql_query, reward=reward, done=done, error=error)
216
+
217
+ if done:
218
+ break
219
+
220
+ score = max(rewards) if rewards else 0.0
221
+ score = min(max(score, 0.0), 1.0)
222
+ success = score >= SUCCESS_THRESHOLD
223
+
224
+ except Exception as e:
225
+ print(f"[DEBUG] Task {task_id} error: {e}", flush=True)
226
+ finally:
227
+ log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
228
+
229
+ return score
230
+
231
+
232
+ async def main() -> None:
233
+ """Run all tasks and print summary."""
234
+ client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
235
+
236
+ if IMAGE_NAME:
237
+ env = await SqlQueryEnv.from_docker_image(IMAGE_NAME)
238
+ else:
239
+ base_url = os.getenv("ENV_BASE_URL", "http://localhost:8000")
240
+ env = SqlQueryEnv(base_url=base_url)
241
+
242
+ try:
243
+ async with env:
244
+ scores = []
245
+ for task_id in TASK_IDS:
246
+ score = await run_task(client, env, task_id)
247
+ scores.append(score)
248
+
249
+ # print summary
250
+ print(f"\n{'='*50}", flush=True)
251
+ print(f"[SUMMARY] Model: {MODEL_NAME}", flush=True)
252
+ print(f"[SUMMARY] Average score: {sum(scores)/len(scores):.2f}", flush=True)
253
+ print(f"{'='*50}", flush=True)
254
+
255
+ domains = {"company": [], "hospital": [], "ecommerce": []}
256
+ for tid, sc in zip(TASK_IDS, scores):
257
+ domain = tid.split("_")[0]
258
+ domains[domain].append(sc)
259
+ print(f" {tid}: {sc:.2f}", flush=True)
260
+
261
+ print(f"\nPer-domain averages:", flush=True)
262
+ for domain, dscores in domains.items():
263
+ avg = sum(dscores) / len(dscores) if dscores else 0
264
+ print(f" {domain}: {avg:.2f}", flush=True)
265
+
266
+ except Exception as e:
267
+ print(f"[DEBUG] Environment connection error: {e}", flush=True)
268
+
269
+
270
+ if __name__ == "__main__":
271
+ asyncio.run(main())
models.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Pydantic models for the SQL environment's action/observation space.
3
+
4
+ We tried keeping the observation minimal at first but realized the agent
5
+ needs way more context to actually improve between steps - so we added
6
+ diagnostics, efficiency notes, history etc. Bit of a chonky payload but
7
+ it's all text so doesn't matter much for perf.
8
+ """
9
+
10
+ from typing import Dict, List, Optional
11
+
12
+ from openenv.core.env_server.types import Action, Observation
13
+ from pydantic import Field
14
+
15
+
16
+ class SqlQueryAction(Action):
17
+ """Agent submits a SQL query string."""
18
+
19
+ query: str = Field(
20
+ ...,
21
+ description="SQL query to execute against the database. Must be a valid SELECT statement.",
22
+ )
23
+
24
+
25
+ class SqlQueryObservation(Observation):
26
+ """
27
+ Full observation returned after each step.
28
+
29
+ We pack a lot in here because the agent needs to understand what went
30
+ wrong and how to fix it. The alternative was returning just a score,
31
+ but that gives basically zero learning signal.
32
+ """
33
+
34
+ # task context
35
+ task_id: str = Field(default="", description="Current task identifier")
36
+ difficulty: str = Field(default="easy", description="easy, medium, or hard")
37
+ database_domain: str = Field(
38
+ default="company", description="Which database domain this task uses"
39
+ )
40
+ question: str = Field(default="", description="Natural language question to answer with SQL")
41
+ schema_description: str = Field(
42
+ default="", description="Full database schema with table definitions"
43
+ )
44
+
45
+ # query feedback
46
+ query_result: str = Field(default="", description="Formatted result of the last query")
47
+ query_error: Optional[str] = Field(default=None, description="SQL error message, if any")
48
+ feedback: str = Field(default="", description="Detailed grading feedback")
49
+
50
+ # structured diagnostics - these are the key differentiator
51
+ diagnostics: List[Dict] = Field(
52
+ default_factory=list,
53
+ description="Structured error diagnostics: type, severity, message, suggestion",
54
+ )
55
+ efficiency_notes: List[str] = Field(
56
+ default_factory=list,
57
+ description="SQL best-practice tips based on the submitted query",
58
+ )
59
+
60
+ # hints to help the agent
61
+ expected_row_count: int = Field(default=0, description="Expected number of result rows")
62
+ expected_columns: List[str] = Field(default_factory=list, description="Expected column names")
63
+
64
+ # episode progress
65
+ steps_remaining: int = Field(default=0, description="Attempts left in this episode")
66
+ current_score: float = Field(default=0.0, description="Best score so far (0.0-1.0)")
67
+ history: List[Dict] = Field(default_factory=list, description="Previous queries and scores")
openenv.yaml ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ spec_version: 1
2
+ name: sql_query_env
3
+ description: >
4
+ Trains agents to generate SQL queries from natural language questions.
5
+ Tests across three database domains (company analytics, hospital
6
+ management, e-commerce) with efficiency scoring and diagnostic feedback.
7
+ type: space
8
+ runtime: fastapi
9
+ app: server.app:app
10
+ port: 8000
11
+
12
+ tasks:
13
+ # Company
14
+ - id: company_easy_1
15
+ name: "Company β€” Engineering Employees"
16
+ difficulty: easy
17
+ description: "List active Engineering employees sorted by salary"
18
+ domain: company
19
+
20
+ - id: company_easy_2
21
+ name: "Company β€” Electronics Products"
22
+ difficulty: easy
23
+ description: "Filter electronics products by price"
24
+ domain: company
25
+
26
+ - id: company_medium_1
27
+ name: "Company β€” Revenue by Category"
28
+ difficulty: medium
29
+ description: "Total revenue per product category for completed orders"
30
+ domain: company
31
+
32
+ - id: company_medium_2
33
+ name: "Company β€” Department Salaries"
34
+ difficulty: medium
35
+ description: "Departments with average salary exceeding $100K"
36
+ domain: company
37
+
38
+ - id: company_hard_1
39
+ name: "Company β€” Budget Utilization"
40
+ difficulty: hard
41
+ description: "Department salary-to-budget utilization percentage"
42
+ domain: company
43
+
44
+ # Hospital
45
+ - id: hospital_easy_1
46
+ name: "Hospital β€” Current Patients"
47
+ difficulty: easy
48
+ description: "List patients still admitted (NULL discharge date)"
49
+ domain: hospital
50
+
51
+ - id: hospital_medium_1
52
+ name: "Hospital β€” Medication Costs"
53
+ difficulty: medium
54
+ description: "Total medication cost for admitted patients"
55
+ domain: hospital
56
+
57
+ - id: hospital_hard_1
58
+ name: "Hospital β€” Cross-Ward Prescriptions"
59
+ difficulty: hard
60
+ description: "Doctors prescribing to patients outside their ward"
61
+ domain: hospital
62
+
63
+ # E-Commerce
64
+ - id: ecommerce_easy_1
65
+ name: "E-Commerce β€” Premium Users"
66
+ difficulty: easy
67
+ description: "List premium users with signup details"
68
+ domain: ecommerce
69
+
70
+ - id: ecommerce_medium_1
71
+ name: "E-Commerce β€” Seller Revenue"
72
+ difficulty: medium
73
+ description: "Revenue per seller for delivered orders"
74
+ domain: ecommerce
75
+
76
+ - id: ecommerce_hard_1
77
+ name: "E-Commerce β€” Return Rate Analysis"
78
+ difficulty: hard
79
+ description: "Return rate per product category with edge cases"
80
+ domain: ecommerce
81
+
82
+ # Expert-level
83
+ - id: company_hard_2
84
+ name: "Company β€” Salary vs Manager"
85
+ difficulty: hard
86
+ description: "Employees earning more than their direct manager (self-join)"
87
+ domain: company
88
+
89
+ - id: hospital_hard_2
90
+ name: "Hospital β€” Repeat Visits"
91
+ difficulty: hard
92
+ description: "Patients with multiple appointments with the same doctor"
93
+ domain: hospital
94
+
95
+ - id: ecommerce_hard_2
96
+ name: "E-Commerce β€” Dead Stock"
97
+ difficulty: hard
98
+ description: "In-stock products that have never been ordered (anti-join)"
99
+ domain: ecommerce
100
+
101
+ action_space:
102
+ type: object
103
+ properties:
104
+ query:
105
+ type: string
106
+ description: "SQL SELECT query to execute against the database"
107
+
108
+ observation_space:
109
+ type: object
110
+ properties:
111
+ task_id:
112
+ type: string
113
+ difficulty:
114
+ type: string
115
+ enum: [easy, medium, hard]
116
+ database_domain:
117
+ type: string
118
+ description: "Company Analytics, Hospital Management, or E-Commerce Platform"
119
+ question:
120
+ type: string
121
+ schema_description:
122
+ type: string
123
+ query_result:
124
+ type: string
125
+ query_error:
126
+ type: string
127
+ feedback:
128
+ type: string
129
+ diagnostics:
130
+ type: array
131
+ description: "Structured diagnostic feedback (type, severity, message, suggestion)"
132
+ efficiency_notes:
133
+ type: array
134
+ description: "SQL efficiency and best-practice feedback"
135
+ expected_row_count:
136
+ type: integer
137
+ expected_columns:
138
+ type: array
139
+ steps_remaining:
140
+ type: integer
141
+ current_score:
142
+ type: number
143
+ history:
144
+ type: array
openenv_sql_query_env.egg-info/PKG-INFO ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Metadata-Version: 2.4
2
+ Name: openenv-sql_query_env
3
+ Version: 0.1.0
4
+ Summary: SQL query generation environment for OpenEnv
5
+ Requires-Python: >=3.10
6
+ License-File: LICENSE
7
+ Requires-Dist: openenv-core[core]>=0.2.2
8
+ Provides-Extra: dev
9
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
10
+ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
11
+ Dynamic: license-file
openenv_sql_query_env.egg-info/SOURCES.txt ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ LICENSE
2
+ README.md
3
+ __init__.py
4
+ client.py
5
+ inference.py
6
+ models.py
7
+ pyproject.toml
8
+ ./__init__.py
9
+ ./client.py
10
+ ./inference.py
11
+ ./models.py
12
+ openenv_sql_query_env.egg-info/PKG-INFO
13
+ openenv_sql_query_env.egg-info/SOURCES.txt
14
+ openenv_sql_query_env.egg-info/dependency_links.txt
15
+ openenv_sql_query_env.egg-info/entry_points.txt
16
+ openenv_sql_query_env.egg-info/requires.txt
17
+ openenv_sql_query_env.egg-info/top_level.txt
18
+ server/__init__.py
19
+ server/app.py
20
+ server/graders.py
21
+ server/sql_query_env_environment.py
22
+ server/tasks.py
openenv_sql_query_env.egg-info/dependency_links.txt ADDED
@@ -0,0 +1 @@
 
 
1
+
openenv_sql_query_env.egg-info/entry_points.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ [console_scripts]
2
+ server = sql_query_env.server.app:main
openenv_sql_query_env.egg-info/requires.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ openenv-core[core]>=0.2.2
2
+
3
+ [dev]
4
+ pytest>=8.0.0
5
+ pytest-cov>=4.0.0
openenv_sql_query_env.egg-info/top_level.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ sql_query_env
pyproject.toml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=45", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "openenv-sql_query_env"
7
+ version = "0.1.0"
8
+ description = "SQL query generation environment for OpenEnv"
9
+ requires-python = ">=3.10"
10
+ dependencies = [
11
+ "openenv-core[core]>=0.2.2",
12
+ ]
13
+
14
+ [project.optional-dependencies]
15
+ dev = [
16
+ "pytest>=8.0.0",
17
+ "pytest-cov>=4.0.0",
18
+ ]
19
+
20
+ [project.scripts]
21
+ server = "sql_query_env.server.app:main"
22
+
23
+ [tool.setuptools]
24
+ include-package-data = true
25
+ packages = ["sql_query_env", "sql_query_env.server"]
26
+ package-dir = { "sql_query_env" = ".", "sql_query_env.server" = "server" }
server/Dockerfile ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-stage build using openenv-base
2
+ # This Dockerfile is flexible and works for both:
3
+ # - In-repo environments (with local OpenEnv sources)
4
+ # - Standalone environments (with openenv from PyPI/Git)
5
+ # The build script (openenv build) handles context detection and sets appropriate build args.
6
+
7
+ ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
8
+ FROM ${BASE_IMAGE} AS builder
9
+
10
+ WORKDIR /app
11
+
12
+ # Ensure git is available (required for installing dependencies from VCS)
13
+ RUN apt-get update && \
14
+ apt-get install -y --no-install-recommends git && \
15
+ rm -rf /var/lib/apt/lists/*
16
+
17
+ # Build argument to control whether we're building standalone or in-repo
18
+ ARG BUILD_MODE=in-repo
19
+ ARG ENV_NAME=sql_query_env
20
+
21
+ # Copy environment code (always at root of build context)
22
+ COPY . /app/env
23
+
24
+ # For in-repo builds, openenv is already vendored in the build context
25
+ # For standalone builds, openenv will be installed via pyproject.toml
26
+ WORKDIR /app/env
27
+
28
+ # Ensure uv is available (for local builds where base image lacks it)
29
+ RUN if ! command -v uv >/dev/null 2>&1; then \
30
+ curl -LsSf https://astral.sh/uv/install.sh | sh && \
31
+ mv /root/.local/bin/uv /usr/local/bin/uv && \
32
+ mv /root/.local/bin/uvx /usr/local/bin/uvx; \
33
+ fi
34
+
35
+ # Install dependencies using uv sync
36
+ # If uv.lock exists, use it; otherwise resolve on the fly
37
+ RUN --mount=type=cache,target=/root/.cache/uv \
38
+ if [ -f uv.lock ]; then \
39
+ uv sync --frozen --no-install-project --no-editable; \
40
+ else \
41
+ uv sync --no-install-project --no-editable; \
42
+ fi
43
+
44
+ RUN --mount=type=cache,target=/root/.cache/uv \
45
+ if [ -f uv.lock ]; then \
46
+ uv sync --frozen --no-editable; \
47
+ else \
48
+ uv sync --no-editable; \
49
+ fi
50
+
51
+ # Final runtime stage
52
+ FROM ${BASE_IMAGE}
53
+
54
+ WORKDIR /app
55
+
56
+ # Copy the virtual environment from builder
57
+ COPY --from=builder /app/env/.venv /app/.venv
58
+
59
+ # Copy the environment code
60
+ COPY --from=builder /app/env /app/env
61
+
62
+ # Set PATH to use the virtual environment
63
+ ENV PATH="/app/.venv/bin:$PATH"
64
+
65
+ # Set PYTHONPATH so imports work correctly
66
+ ENV PYTHONPATH="/app/env:$PYTHONPATH"
67
+
68
+ # Health check
69
+ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
70
+ CMD curl -f http://localhost:8000/health || exit 1
71
+
72
+ # Run the FastAPI server
73
+ # The module path is constructed to work with the /app/env structure
74
+ CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]
server/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """SQL Query Environment Server."""
2
+
3
+ from .sql_query_env_environment import SqlQueryEnvironment
4
+
5
+ __all__ = ["SqlQueryEnvironment"]
server/app.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ FastAPI server entry point.
3
+
4
+ Start with:
5
+ uv run --project . server
6
+ uvicorn server.app:app --host 0.0.0.0 --port 8000
7
+ """
8
+
9
+ try:
10
+ from openenv.core.env_server.http_server import create_app
11
+ except Exception as e:
12
+ raise ImportError(
13
+ "openenv-core is required. Install with: uv sync"
14
+ ) from e
15
+
16
+ # dual-import fallback for different run modes
17
+ try:
18
+ from ..models import SqlQueryAction, SqlQueryObservation
19
+ from .sql_query_env_environment import SqlQueryEnvironment
20
+ except (ImportError, ModuleNotFoundError):
21
+ from models import SqlQueryAction, SqlQueryObservation
22
+ from server.sql_query_env_environment import SqlQueryEnvironment
23
+
24
+
25
+ app = create_app(
26
+ SqlQueryEnvironment,
27
+ SqlQueryAction,
28
+ SqlQueryObservation,
29
+ env_name="sql_query_env",
30
+ max_concurrent_envs=256,
31
+ )
32
+
33
+
34
+ def main():
35
+ """Entry point for `uv run server` and `openenv serve`."""
36
+ import uvicorn
37
+ uvicorn.run(app, host="0.0.0.0", port=8000)
38
+
39
+
40
+ if __name__ == "__main__":
41
+ main()
server/graders.py ADDED
@@ -0,0 +1,593 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Grading logic for SQL queries.
3
+
4
+ The scoring is split into five components instead of a binary pass/fail.
5
+ We found during testing that a query referencing the right tables but
6
+ messing up column names should still get partial credit β€” otherwise the
7
+ agent gets zero signal on almost-correct attempts and can't learn.
8
+
9
+ Score weights were chosen by trial and error:
10
+ syntax=0.10, tables=0.15, columns=0.20, results=0.45, efficiency=0.10
11
+
12
+ A query needs to be mostly correct to break 0.70 but you always get
13
+ *something* for partial progress.
14
+ """
15
+
16
+ import re
17
+ import sqlite3
18
+ from dataclasses import dataclass, field
19
+ from typing import Dict, List, Optional, Set, Tuple
20
+
21
+ from .tasks import Task
22
+
23
+
24
+ # ---------------------------------------------------------------------------
25
+ # Data structures
26
+ # ---------------------------------------------------------------------------
27
+
28
+ @dataclass
29
+ class Diagnostic:
30
+ """Structured diagnostic for process supervision feedback."""
31
+ type: str # e.g. MISSING_JOIN, WRONG_TABLE
32
+ severity: str # error / warning / info
33
+ message: str
34
+ suggestion: str
35
+
36
+ def to_dict(self) -> Dict:
37
+ return {
38
+ "type": self.type,
39
+ "severity": self.severity,
40
+ "message": self.message,
41
+ "suggestion": self.suggestion,
42
+ }
43
+
44
+
45
+ @dataclass
46
+ class GradeResult:
47
+ """Holds all scoring info for a single query attempt."""
48
+
49
+ total_score: float
50
+ syntax_score: float
51
+ table_score: float
52
+ column_score: float
53
+ result_score: float
54
+ efficiency_score: float
55
+ penalty: float
56
+ feedback: str
57
+ query_result: Optional[str]
58
+ error: Optional[str]
59
+ rows_returned: int
60
+ rows_expected: int
61
+ diagnostics: List[Diagnostic] = field(default_factory=list)
62
+ efficiency_notes: List[str] = field(default_factory=list)
63
+
64
+
65
+ # ---------------------------------------------------------------------------
66
+ # Helper functions
67
+ # ---------------------------------------------------------------------------
68
+
69
+ def _extract_tables(query: str) -> Set[str]:
70
+ """Pull table names out of FROM and JOIN clauses.
71
+
72
+ Quick regex approach β€” doesn't handle subquery aliases or CTEs but
73
+ it's good enough for our task set. Didn't want to pull in sqlparse
74
+ just for this.
75
+ """
76
+ q = " ".join(query.lower().split())
77
+ tables = set()
78
+ for match in re.finditer(r'\bfrom\s+(\w+)', q):
79
+ tables.add(match.group(1))
80
+ for match in re.finditer(r'\bjoin\s+(\w+)', q):
81
+ tables.add(match.group(1))
82
+ return tables
83
+
84
+
85
+ def _format_result(rows: List[Tuple], columns: List[str], max_rows: int = 20) -> str:
86
+ """Pretty-print query results."""
87
+ if not rows:
88
+ return "(no rows returned)"
89
+
90
+ lines = []
91
+ header = " | ".join(columns)
92
+ lines.append(header)
93
+ lines.append("-" * len(header))
94
+
95
+ for row in rows[:max_rows]:
96
+ lines.append(" | ".join(str(v) for v in row))
97
+ if len(rows) > max_rows:
98
+ lines.append(f"... ({len(rows) - max_rows} more rows)")
99
+
100
+ return "\n".join(lines)
101
+
102
+
103
+ def _is_destructive(query: str) -> bool:
104
+ """Check if query tries to modify data. We block everything except SELECT.
105
+
106
+ Double-checking here even though the environment also uses read-only
107
+ connections β€” better safe than sorry.
108
+ """
109
+ q = query.upper().strip()
110
+ bad_keywords = ["DROP", "DELETE", "TRUNCATE", "ALTER", "UPDATE", "INSERT", "CREATE"]
111
+ for kw in bad_keywords:
112
+ if re.search(rf'\b{kw}\b', q):
113
+ return True
114
+ return False
115
+
116
+
117
+ def _normalize_value(v) -> str:
118
+ """Normalize cell values for comparison.
119
+
120
+ Rounds floats to 2 decimal places so 3.1000001 matches 3.10.
121
+ """
122
+ if v is None:
123
+ return "NULL"
124
+ if isinstance(v, float):
125
+ return f"{v:.2f}"
126
+ return str(v).strip().lower()
127
+
128
+
129
+ def _compare_result_sets(
130
+ actual_rows: List[Tuple],
131
+ expected_rows: List[Tuple],
132
+ query: str = "",
133
+ ground_truth_query: str = "",
134
+ ) -> float:
135
+ """Compare actual vs expected results, returns similarity 0-1.
136
+
137
+ Also checks row ordering if the ground truth has ORDER BY.
138
+ """
139
+ if not expected_rows:
140
+ return 1.0 if not actual_rows else 0.0
141
+ if not actual_rows:
142
+ return 0.0
143
+
144
+ def normalize_row(row: Tuple) -> Tuple[str, ...]:
145
+ return tuple(_normalize_value(v) for v in row)
146
+
147
+ actual_norm = [normalize_row(r) for r in actual_rows]
148
+ expected_norm = [normalize_row(r) for r in expected_rows]
149
+
150
+ actual_set = set(actual_norm)
151
+ expected_set = set(expected_norm)
152
+
153
+ matching = actual_set & expected_set
154
+ total_expected = len(expected_set)
155
+
156
+ if total_expected == 0:
157
+ return 1.0
158
+
159
+ # what fraction of expected rows did we get?
160
+ content_score = len(matching) / total_expected
161
+
162
+ # dock points for extra rows the agent shouldn't have returned
163
+ extra = len(actual_set - expected_set)
164
+ if extra > 0:
165
+ content_score = max(0.0, content_score - min(extra / total_expected * 0.3, 0.3))
166
+
167
+ # bonus for correct ordering when ORDER BY is expected
168
+ order_bonus = 0.0
169
+ has_order = "ORDER BY" in ground_truth_query.upper()
170
+ if has_order and content_score > 0.5 and len(actual_norm) == len(expected_norm):
171
+ correct_positions = sum(
172
+ 1 for a, e in zip(actual_norm, expected_norm) if a == e
173
+ )
174
+ order_bonus = (correct_positions / len(expected_norm)) * 0.1
175
+
176
+ return min(content_score + order_bonus, 1.0)
177
+
178
+
179
+ # ---------------------------------------------------------------------------
180
+ # Diagnostic engine
181
+ # ---------------------------------------------------------------------------
182
+
183
+ def _diagnose_query(
184
+ query: str,
185
+ task: Task,
186
+ actual_rows: List[Tuple],
187
+ expected_rows: List[Tuple],
188
+ actual_cols: List[str],
189
+ expected_cols: List[str],
190
+ error: Optional[str],
191
+ ) -> List[Diagnostic]:
192
+ """Figure out what went wrong and give actionable feedback.
193
+
194
+ We check for ~17 different issue types. The goal is process
195
+ supervision: give the agent enough info to fix its query on
196
+ the next attempt.
197
+ """
198
+ diagnostics = []
199
+ q_upper = query.upper()
200
+ gt_upper = task.ground_truth_query.upper()
201
+
202
+ # -- syntax errors (parsed from SQLite error messages) --
203
+ if error:
204
+ if "no such table" in error.lower():
205
+ table_match = re.search(r'no such table: (\w+)', error, re.IGNORECASE)
206
+ table_name = table_match.group(1) if table_match else "unknown"
207
+ diagnostics.append(Diagnostic(
208
+ type="WRONG_TABLE_NAME",
209
+ severity="error",
210
+ message=f"Table '{table_name}' does not exist in this database.",
211
+ suggestion="Check the schema description for available table names.",
212
+ ))
213
+ elif "no such column" in error.lower():
214
+ col_match = re.search(r'no such column: (\S+)', error, re.IGNORECASE)
215
+ col_name = col_match.group(1) if col_match else "unknown"
216
+ diagnostics.append(Diagnostic(
217
+ type="WRONG_COLUMN_NAME",
218
+ severity="error",
219
+ message=f"Column '{col_name}' does not exist.",
220
+ suggestion="Check the table definition for correct column names.",
221
+ ))
222
+ elif "ambiguous column" in error.lower():
223
+ diagnostics.append(Diagnostic(
224
+ type="AMBIGUOUS_COLUMN",
225
+ severity="error",
226
+ message="A column name is ambiguous - it exists in multiple tables.",
227
+ suggestion="Use table aliases (e.g. e.name instead of just name).",
228
+ ))
229
+ else:
230
+ diagnostics.append(Diagnostic(
231
+ type="SYNTAX_ERROR",
232
+ severity="error",
233
+ message=f"SQL syntax error: {error[:100]}",
234
+ suggestion="Check your SQL syntax. Common issues: missing commas, unmatched parentheses, typos.",
235
+ ))
236
+ return diagnostics
237
+
238
+ # -- structural checks --
239
+ expected_tables = _extract_tables(task.ground_truth_query)
240
+ actual_tables = _extract_tables(query)
241
+
242
+ missing_tables = expected_tables - actual_tables
243
+ if missing_tables:
244
+ diagnostics.append(Diagnostic(
245
+ type="MISSING_TABLE",
246
+ severity="warning",
247
+ message=f"Your query doesn't reference: {', '.join(missing_tables)}",
248
+ suggestion=f"You probably need to JOIN with {', '.join(missing_tables)} to get the required data.",
249
+ ))
250
+
251
+ if "JOIN" in gt_upper and "JOIN" not in q_upper:
252
+ diagnostics.append(Diagnostic(
253
+ type="MISSING_JOIN",
254
+ severity="warning",
255
+ message="This question requires data from multiple tables, but no JOIN was found.",
256
+ suggestion="Use JOIN to combine tables. Example: SELECT ... FROM t1 JOIN t2 ON t1.id = t2.t1_id",
257
+ ))
258
+
259
+ if "GROUP BY" in gt_upper and "GROUP BY" not in q_upper:
260
+ diagnostics.append(Diagnostic(
261
+ type="MISSING_AGGREGATION",
262
+ severity="warning",
263
+ message="This question requires grouping/aggregation, but no GROUP BY was found.",
264
+ suggestion="Use GROUP BY with aggregate functions (SUM, COUNT, AVG) to group results.",
265
+ ))
266
+
267
+ if "HAVING" in gt_upper and "HAVING" not in q_upper and "GROUP BY" in q_upper:
268
+ diagnostics.append(Diagnostic(
269
+ type="MISSING_HAVING",
270
+ severity="info",
271
+ message="You might need a HAVING clause to filter grouped results.",
272
+ suggestion="HAVING filters after GROUP BY. Example: HAVING COUNT(*) > 5",
273
+ ))
274
+
275
+ if "IS NULL" in gt_upper and "IS NULL" not in q_upper:
276
+ diagnostics.append(Diagnostic(
277
+ type="NULL_HANDLING",
278
+ severity="warning",
279
+ message="This question involves NULL values. Use IS NULL or IS NOT NULL.",
280
+ suggestion="In SQL, NULL != NULL. Use IS NULL to check for missing values, not = NULL.",
281
+ ))
282
+
283
+ # -- result set checks --
284
+ missing_cols = set(expected_cols) - set(actual_cols)
285
+ if missing_cols:
286
+ diagnostics.append(Diagnostic(
287
+ type="MISSING_COLUMNS",
288
+ severity="warning",
289
+ message=f"Missing output columns: {', '.join(missing_cols)}",
290
+ suggestion=f"Add these to your SELECT: {', '.join(missing_cols)}",
291
+ ))
292
+
293
+ extra_cols = set(actual_cols) - set(expected_cols)
294
+ if extra_cols:
295
+ diagnostics.append(Diagnostic(
296
+ type="EXTRA_COLUMNS",
297
+ severity="info",
298
+ message=f"Extra columns in output: {', '.join(extra_cols)}",
299
+ suggestion=f"Expected columns are: {', '.join(expected_cols)}",
300
+ ))
301
+
302
+ if len(actual_rows) > len(expected_rows):
303
+ diagnostics.append(Diagnostic(
304
+ type="EXTRA_ROWS",
305
+ severity="warning",
306
+ message=f"Got {len(actual_rows)} rows but expected {len(expected_rows)}.",
307
+ suggestion="Your WHERE/HAVING conditions may be too broad. Try adding more filters.",
308
+ ))
309
+ elif len(actual_rows) < len(expected_rows) and len(actual_rows) > 0:
310
+ diagnostics.append(Diagnostic(
311
+ type="MISSING_ROWS",
312
+ severity="warning",
313
+ message=f"Got {len(actual_rows)} rows but expected {len(expected_rows)}.",
314
+ suggestion="Filters might be too restrictive, or try LEFT JOIN instead of INNER JOIN.",
315
+ ))
316
+
317
+ return diagnostics
318
+
319
+
320
+ # ---------------------------------------------------------------------------
321
+ # Efficiency scoring
322
+ # ---------------------------------------------------------------------------
323
+
324
+ def _score_efficiency(
325
+ query: str,
326
+ task: Task,
327
+ actual_cols: List[str],
328
+ expected_cols: List[str],
329
+ ) -> Tuple[float, List[str]]:
330
+ """Score SQL best practices (0.0 - 0.10).
331
+
332
+ Rewards good habits and penalizes bad ones. The weights are small
333
+ (max 0.10 total) so this doesn't dominate scoring, but it adds up
334
+ and teaches the agent to write cleaner SQL.
335
+ """
336
+ notes = []
337
+ score = 0.0
338
+ q_upper = query.upper().strip()
339
+
340
+ # -- good practices --
341
+ if re.search(r'\bAS\s+\w+', query, re.IGNORECASE):
342
+ score += 0.03
343
+ notes.append("Good: using column aliases (AS)")
344
+
345
+ if "SELECT *" not in q_upper and len(expected_cols) > 0:
346
+ if set(actual_cols) == set(expected_cols) or set(actual_cols).issuperset(set(expected_cols)):
347
+ score += 0.02
348
+ notes.append("Good: selecting specific columns instead of SELECT *")
349
+
350
+ alias_pattern = r'\b(?:FROM|JOIN)\s+\w+\s+(?:AS\s+)?([a-z]\w{0,2})\b'
351
+ if re.search(alias_pattern, query, re.IGNORECASE):
352
+ score += 0.02
353
+ notes.append("Good: using table aliases for readability")
354
+
355
+ if re.search(r'\bCOALESCE\b|\bIFNULL\b', query, re.IGNORECASE):
356
+ score += 0.03
357
+ notes.append("Good: proper NULL handling with COALESCE/IFNULL")
358
+
359
+ # -- bad practices --
360
+ if "SELECT *" in q_upper and len(expected_cols) > 0 and len(expected_cols) < 8:
361
+ score -= 0.02
362
+ notes.append("Avoid SELECT * when you only need specific columns")
363
+
364
+ if "DISTINCT" in q_upper and "DISTINCT" not in task.ground_truth_query.upper():
365
+ score -= 0.01
366
+ notes.append("DISTINCT may not be needed here - check if duplicates are actually expected")
367
+
368
+ # check for cartesian product (FROM a, b without JOIN)
369
+ from_tables = re.findall(r'\bFROM\s+(\w+(?:\s*,\s*\w+)+)', query, re.IGNORECASE)
370
+ if from_tables and "JOIN" not in q_upper:
371
+ score -= 0.02
372
+ notes.append("Warning: potential Cartesian product - use explicit JOIN instead of comma-separated tables")
373
+
374
+ score = max(0.0, min(score, 0.10))
375
+ return score, notes
376
+
377
+
378
+ # ---------------------------------------------------------------------------
379
+ # Main grading function
380
+ # ---------------------------------------------------------------------------
381
+
382
+ def grade_query(
383
+ conn: sqlite3.Connection,
384
+ query: str,
385
+ task: Task,
386
+ previous_queries: List[str],
387
+ ) -> GradeResult:
388
+ """Grade a submitted SQL query.
389
+
390
+ Scoring breakdown:
391
+ syntax: 0.10 (query runs without error)
392
+ tables: 0.15 (correct tables referenced)
393
+ columns: 0.20 (correct output columns)
394
+ results: 0.45 (correct data returned)
395
+ efficiency: 0.10 (query quality)
396
+ penalties: -0.10 for destructive SQL, -0.05 for repeated queries
397
+ """
398
+ feedback_parts = []
399
+ syntax_score = 0.0
400
+ table_score = 0.0
401
+ column_score = 0.0
402
+ result_score = 0.0
403
+ efficiency_score = 0.0
404
+ penalty = 0.0
405
+ query_result_str = None
406
+ error_msg = None
407
+ rows_returned = 0
408
+ rows_expected = 0
409
+ diagnostics = []
410
+ efficiency_notes = []
411
+
412
+ # check for destructive queries
413
+ if _is_destructive(query):
414
+ penalty -= 0.10
415
+ feedback_parts.append("PENALTY: Destructive SQL detected. Only SELECT statements are allowed.")
416
+ diagnostics.append(Diagnostic(
417
+ type="DESTRUCTIVE_QUERY",
418
+ severity="error",
419
+ message="Destructive operations (DROP, DELETE, etc.) are not allowed.",
420
+ suggestion="Use only SELECT statements to query data.",
421
+ ))
422
+
423
+ # check for repeated query
424
+ normalized_query = " ".join(query.lower().split())
425
+ for prev in previous_queries:
426
+ if " ".join(prev.lower().split()) == normalized_query:
427
+ penalty -= 0.05
428
+ feedback_parts.append("PENALTY: Identical query submitted before. Try something different.")
429
+ diagnostics.append(Diagnostic(
430
+ type="REPEATED_QUERY",
431
+ severity="warning",
432
+ message="This exact query was already submitted.",
433
+ suggestion="Modify your query based on the previous feedback to improve.",
434
+ ))
435
+ break
436
+
437
+ # 1. Syntax check (0.10)
438
+ actual_rows = []
439
+ actual_cols = []
440
+ try:
441
+ cursor = conn.execute(query)
442
+ actual_cols = [desc[0].lower() for desc in cursor.description] if cursor.description else []
443
+ actual_rows = cursor.fetchall()
444
+ rows_returned = len(actual_rows)
445
+ syntax_score = 0.10
446
+ feedback_parts.append("Query executed successfully.")
447
+ query_result_str = _format_result(actual_rows, actual_cols)
448
+ except Exception as e:
449
+ error_msg = str(e)
450
+ feedback_parts.append(f"SQL Error: {error_msg}")
451
+
452
+ diagnostics = _diagnose_query(
453
+ query, task, [], [], [], [], error_msg
454
+ )
455
+
456
+ total = max(0.0, penalty)
457
+ return GradeResult(
458
+ total_score=total,
459
+ syntax_score=0.0,
460
+ table_score=0.0,
461
+ column_score=0.0,
462
+ result_score=0.0,
463
+ efficiency_score=0.0,
464
+ penalty=penalty,
465
+ feedback="\n".join(feedback_parts),
466
+ query_result=None,
467
+ error=error_msg,
468
+ rows_returned=0,
469
+ rows_expected=0,
470
+ diagnostics=diagnostics,
471
+ efficiency_notes=[],
472
+ )
473
+
474
+ # get ground truth results
475
+ try:
476
+ expected_cursor = conn.execute(task.ground_truth_query)
477
+ expected_rows = expected_cursor.fetchall()
478
+ expected_cols_from_query = [
479
+ desc[0].lower() for desc in expected_cursor.description
480
+ ] if expected_cursor.description else []
481
+ rows_expected = len(expected_rows)
482
+ except Exception:
483
+ expected_rows = []
484
+ expected_cols_from_query = []
485
+ rows_expected = 0
486
+
487
+ # 2. Table references (0.15)
488
+ expected_tables = _extract_tables(task.ground_truth_query)
489
+ actual_tables = _extract_tables(query)
490
+
491
+ if expected_tables:
492
+ matching_tables = actual_tables & expected_tables
493
+ table_ratio = len(matching_tables) / len(expected_tables)
494
+ table_score = 0.15 * table_ratio
495
+
496
+ if table_ratio == 1.0:
497
+ feedback_parts.append("Correct tables referenced.")
498
+ elif table_ratio > 0:
499
+ missing = expected_tables - actual_tables
500
+ feedback_parts.append(f"Partially correct tables. Missing: {', '.join(missing)}")
501
+ else:
502
+ feedback_parts.append(f"Wrong tables. Consider using: {', '.join(expected_tables)}")
503
+
504
+ # 3. Column check (0.20)
505
+ expected_cols_lower = [c.lower() for c in task.expected_columns]
506
+
507
+ if expected_cols_lower:
508
+ matching_cols = set(actual_cols) & set(expected_cols_lower)
509
+ col_ratio = len(matching_cols) / len(expected_cols_lower)
510
+ column_score = 0.20 * col_ratio
511
+
512
+ if col_ratio == 1.0:
513
+ feedback_parts.append("Correct output columns.")
514
+ elif col_ratio > 0:
515
+ missing_cols = set(expected_cols_lower) - set(actual_cols)
516
+ feedback_parts.append(f"Missing columns: {', '.join(missing_cols)}")
517
+ else:
518
+ feedback_parts.append(
519
+ f"Wrong columns. Expected: {', '.join(expected_cols_lower)}. Got: {', '.join(actual_cols)}"
520
+ )
521
+
522
+ # 4. Result set comparison (0.45)
523
+ similarity = _compare_result_sets(
524
+ actual_rows, expected_rows, query, task.ground_truth_query
525
+ )
526
+ result_score = 0.45 * similarity
527
+
528
+ if similarity == 1.0:
529
+ feedback_parts.append("Perfect result set - all rows match.")
530
+ elif similarity > 0.5:
531
+ feedback_parts.append(
532
+ f"Partial match: {similarity:.0%} of expected rows. "
533
+ f"Got {rows_returned} rows, expected {rows_expected}."
534
+ )
535
+ elif similarity > 0:
536
+ feedback_parts.append(
537
+ f"Low match: {similarity:.0%} of expected rows. "
538
+ f"Got {rows_returned} rows, expected {rows_expected}."
539
+ )
540
+ else:
541
+ feedback_parts.append(
542
+ f"No matching rows. Expected {rows_expected} rows, got {rows_returned}."
543
+ )
544
+
545
+ # 5. Efficiency (0.10)
546
+ efficiency_score, efficiency_notes = _score_efficiency(
547
+ query, task, actual_cols, expected_cols_lower
548
+ )
549
+
550
+ # generate diagnostics
551
+ diagnostics = _diagnose_query(
552
+ query, task, actual_rows, expected_rows,
553
+ actual_cols, expected_cols_lower, None,
554
+ )
555
+
556
+ # compute final score
557
+ total = syntax_score + table_score + column_score + result_score + efficiency_score + penalty
558
+ total = max(0.0, min(total, 1.0))
559
+
560
+ # score breakdown
561
+ feedback_parts.append(
562
+ f"\nScore: syntax={syntax_score:.2f} + tables={table_score:.2f} + "
563
+ f"columns={column_score:.2f} + results={result_score:.2f} + "
564
+ f"efficiency={efficiency_score:.2f} + penalties={penalty:.2f} = {total:.2f}"
565
+ )
566
+
567
+ if efficiency_notes:
568
+ feedback_parts.append("\nQuery quality notes:")
569
+ feedback_parts.extend(f" - {note}" for note in efficiency_notes)
570
+
571
+ if diagnostics:
572
+ feedback_parts.append("\nIssues found:")
573
+ for d in diagnostics:
574
+ severity_tag = "ERR" if d.severity == "error" else "WARN" if d.severity == "warning" else "INFO"
575
+ feedback_parts.append(f" [{severity_tag}] {d.type}: {d.message}")
576
+ feedback_parts.append(f" Suggestion: {d.suggestion}")
577
+
578
+ return GradeResult(
579
+ total_score=total,
580
+ syntax_score=syntax_score,
581
+ table_score=table_score,
582
+ column_score=column_score,
583
+ result_score=result_score,
584
+ efficiency_score=efficiency_score,
585
+ penalty=penalty,
586
+ feedback="\n".join(feedback_parts),
587
+ query_result=query_result_str,
588
+ error=error_msg,
589
+ rows_returned=rows_returned,
590
+ rows_expected=rows_expected,
591
+ diagnostics=diagnostics,
592
+ efficiency_notes=efficiency_notes,
593
+ )
server/requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ openenv[core]>=0.2.0
2
+ fastapi>=0.115.0
3
+ uvicorn>=0.24.0
4
+
5
+
6
+
server/sql_query_env_environment.py ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Core environment logic for the SQL query environment.
3
+
4
+ Each episode creates a fresh in-memory SQLite database for the chosen
5
+ domain so there's no state leaking between episodes. SQLite in-memory
6
+ databases are cheap to spin up so this doesn't hurt performance.
7
+
8
+ TODO(team): might want to add a "curriculum mode" that auto-selects
9
+ tasks of increasing difficulty within a session. For now we just let
10
+ the caller pick or we cycle through randomly.
11
+ """
12
+
13
+ import random
14
+ import sqlite3
15
+ from typing import Dict, List, Optional
16
+ from uuid import uuid4
17
+
18
+ from openenv.core.env_server.interfaces import Environment
19
+ from openenv.core.env_server.types import State
20
+
21
+ # Handle both import paths β€” running via `uv run server` uses a different
22
+ # package layout than `python -m sql_query_env.server.app`
23
+ try:
24
+ from ..models import SqlQueryAction, SqlQueryObservation
25
+ except ImportError:
26
+ from models import SqlQueryAction, SqlQueryObservation
27
+
28
+ try:
29
+ from .tasks import (
30
+ ALL_TASKS,
31
+ ALL_TASKS_LIST,
32
+ SCHEMAS,
33
+ TASK_MAP,
34
+ Task,
35
+ create_database,
36
+ get_expected_result,
37
+ )
38
+ from .graders import grade_query, GradeResult
39
+ except ImportError:
40
+ from server.tasks import (
41
+ ALL_TASKS,
42
+ ALL_TASKS_LIST,
43
+ SCHEMAS,
44
+ TASK_MAP,
45
+ Task,
46
+ create_database,
47
+ get_expected_result,
48
+ )
49
+ from server.graders import grade_query, GradeResult
50
+
51
+
52
+ class SqlQueryEnvironment(Environment):
53
+ """Main environment class - presents schema + question, grades SQL.
54
+
55
+ Steps are independent in terms of DB state (no writes accumulate).
56
+ We track query history and best score across steps for penalty logic
57
+ and progressive hints.
58
+ """
59
+
60
+ SUPPORTS_CONCURRENT_SESSIONS: bool = True
61
+
62
+ def __init__(self):
63
+ self._state = State(episode_id=str(uuid4()), step_count=0)
64
+ self._db: Optional[sqlite3.Connection] = None
65
+ self._current_task: Optional[Task] = None
66
+ self._current_schema_id: Optional[str] = None
67
+ self._previous_queries: List[str] = []
68
+ self._best_score: float = 0.0
69
+ self._step_rewards: List[float] = []
70
+ self._task_queue: List[str] = []
71
+ self._current_task_index: int = 0
72
+
73
+ def _get_task_list(self) -> List[str]:
74
+ """Shuffle task IDs for random ordering."""
75
+ all_ids = [t.id for t in ALL_TASKS_LIST]
76
+ random.shuffle(all_ids)
77
+ return all_ids
78
+
79
+ def _select_task(self, task_id: Optional[str] = None) -> Task:
80
+ """Pick a specific task or grab the next one from the queue."""
81
+ if task_id and task_id in TASK_MAP:
82
+ return TASK_MAP[task_id]
83
+
84
+ # lazy-init the queue
85
+ if not self._task_queue:
86
+ self._task_queue = self._get_task_list()
87
+ self._current_task_index = 0
88
+
89
+ if self._current_task_index >= len(self._task_queue):
90
+ self._current_task_index = 0
91
+
92
+ task = TASK_MAP[self._task_queue[self._current_task_index]]
93
+ self._current_task_index += 1
94
+ return task
95
+
96
+ def reset(self, task_id: Optional[str] = None) -> SqlQueryObservation:
97
+ """Start a new episode.
98
+
99
+ Creates a fresh DB for the task's domain and returns the initial
100
+ observation with schema, question, and hints.
101
+ """
102
+ self._state = State(episode_id=str(uuid4()), step_count=0)
103
+ self._previous_queries = []
104
+ self._best_score = 0.0
105
+ self._step_rewards = []
106
+
107
+ self._current_task = self._select_task(task_id)
108
+ self._current_schema_id = self._current_task.schema_id
109
+
110
+ # close old connection if any
111
+ if self._db:
112
+ try:
113
+ self._db.close()
114
+ except Exception:
115
+ pass
116
+ self._db = create_database(self._current_schema_id)
117
+
118
+ schema = SCHEMAS[self._current_schema_id]
119
+ expected_rows = get_expected_result(self._db, self._current_task.ground_truth_query)
120
+
121
+ return SqlQueryObservation(
122
+ task_id=self._current_task.id,
123
+ difficulty=self._current_task.difficulty,
124
+ database_domain=schema.name,
125
+ question=self._current_task.question,
126
+ schema_description=schema.description,
127
+ query_result="",
128
+ query_error=None,
129
+ feedback=(
130
+ f"Task: {self._current_task.description}\n"
131
+ f"Database: {schema.name}\n"
132
+ f"Difficulty: {self._current_task.difficulty}\n"
133
+ f"Attempts available: {self._current_task.max_steps}\n"
134
+ f"\nHints:\n" + "\n".join(f" - {h}" for h in self._current_task.hints[:1])
135
+ ),
136
+ diagnostics=[],
137
+ efficiency_notes=[],
138
+ expected_row_count=len(expected_rows),
139
+ expected_columns=self._current_task.expected_columns,
140
+ steps_remaining=self._current_task.max_steps,
141
+ current_score=0.0,
142
+ history=[],
143
+ done=False,
144
+ reward=0.0,
145
+ )
146
+
147
+ def _progressive_hints(self, step: int) -> str:
148
+ """Reveal more hints as the agent struggles.
149
+
150
+ Step 1: just the first hint
151
+ Step 2-3: a few more
152
+ Step 4+: everything we've got
153
+ """
154
+ task = self._current_task
155
+ hints = task.hints
156
+
157
+ if step <= 1:
158
+ shown = hints[:1] if hints else []
159
+ elif step <= 3:
160
+ shown = hints[:min(step + 1, len(hints))]
161
+ else:
162
+ shown = hints
163
+
164
+ if not shown:
165
+ return ""
166
+ return "\n".join(f" - {h}" for h in shown)
167
+
168
+ def step(self, action: SqlQueryAction) -> SqlQueryObservation:
169
+ """Evaluate a submitted SQL query and return graded feedback."""
170
+ if self._current_task is None or self._db is None:
171
+ return SqlQueryObservation(
172
+ feedback="Environment not initialized. Call reset() first.",
173
+ done=True,
174
+ reward=0.0,
175
+ )
176
+
177
+ self._state.step_count += 1
178
+ step = self._state.step_count
179
+ steps_remaining = self._current_task.max_steps - step
180
+
181
+ # grade it
182
+ grade_result = grade_query(
183
+ conn=self._db,
184
+ query=action.query,
185
+ task=self._current_task,
186
+ previous_queries=self._previous_queries,
187
+ )
188
+
189
+ self._previous_queries.append(action.query)
190
+ self._step_rewards.append(grade_result.total_score)
191
+ self._best_score = max(self._best_score, grade_result.total_score)
192
+
193
+ # done if near-perfect or out of steps
194
+ is_done = grade_result.total_score >= 0.95 or steps_remaining <= 0
195
+
196
+ # build history for the observation
197
+ history = [
198
+ {
199
+ "step": i + 1,
200
+ "query": q[:100] + "..." if len(q) > 100 else q,
201
+ "score": round(s, 2),
202
+ }
203
+ for i, (q, s) in enumerate(
204
+ zip(self._previous_queries, self._step_rewards)
205
+ )
206
+ ]
207
+
208
+ feedback = grade_result.feedback
209
+
210
+ # add hints if the agent is struggling
211
+ if not is_done and grade_result.total_score < 0.95:
212
+ hint_text = self._progressive_hints(step)
213
+ if hint_text:
214
+ feedback += f"\n\nHints (attempt {step}):\n{hint_text}"
215
+
216
+ if is_done:
217
+ if grade_result.total_score >= 0.95:
218
+ feedback += "\n\nNear-perfect score achieved."
219
+ else:
220
+ feedback += f"\n\nEpisode finished. Best score: {self._best_score:.2f}"
221
+ feedback += f"\nScore progression: {' -> '.join(f'{s:.2f}' for s in self._step_rewards)}"
222
+
223
+ expected_rows = get_expected_result(self._db, self._current_task.ground_truth_query)
224
+ schema = SCHEMAS[self._current_schema_id]
225
+
226
+ return SqlQueryObservation(
227
+ task_id=self._current_task.id,
228
+ difficulty=self._current_task.difficulty,
229
+ database_domain=schema.name,
230
+ question=self._current_task.question,
231
+ schema_description=schema.description,
232
+ query_result=grade_result.query_result or "",
233
+ query_error=grade_result.error,
234
+ feedback=feedback,
235
+ diagnostics=[d.to_dict() for d in grade_result.diagnostics],
236
+ efficiency_notes=grade_result.efficiency_notes,
237
+ expected_row_count=len(expected_rows),
238
+ expected_columns=self._current_task.expected_columns,
239
+ steps_remaining=max(0, steps_remaining),
240
+ current_score=self._best_score,
241
+ history=history,
242
+ done=is_done,
243
+ reward=grade_result.total_score,
244
+ metadata={
245
+ "task_id": self._current_task.id,
246
+ "difficulty": self._current_task.difficulty,
247
+ "database_domain": self._current_schema_id,
248
+ "step": step,
249
+ "best_score": self._best_score,
250
+ "syntax_score": grade_result.syntax_score,
251
+ "table_score": grade_result.table_score,
252
+ "column_score": grade_result.column_score,
253
+ "result_score": grade_result.result_score,
254
+ "efficiency_score": grade_result.efficiency_score,
255
+ "penalty": grade_result.penalty,
256
+ "challenge_type": self._current_task.challenge_type,
257
+ },
258
+ )
259
+
260
+ @property
261
+ def state(self) -> State:
262
+ return self._state
263
+
264
+ def close(self):
265
+ if self._db:
266
+ try:
267
+ self._db.close()
268
+ except Exception:
269
+ pass
270
+ self._db = None
server/tasks.py ADDED
@@ -0,0 +1,1068 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Database schemas, seed data, and task definitions.
3
+
4
+ Three separate database domains so the agent can't just memorize one
5
+ schema. We experimented with more domains but three felt like the sweet
6
+ spot for coverage without making the task set unmanageable.
7
+
8
+ Seed data is intentionally small (~10-15 rows per table). Bigger datasets
9
+ would be more realistic but for an RL environment the grading speed matters
10
+ more. We can always scale up the data later if needed.
11
+
12
+ NOTE: if you add a new task, make sure to also add it to the openenv.yaml
13
+ manifest and update the TASK_IDS list in inference.py.
14
+ """
15
+
16
+ import sqlite3
17
+ from dataclasses import dataclass, field
18
+ from typing import Dict, List, Optional, Tuple
19
+
20
+
21
+ # ---------------------------------------------------------------------------
22
+ # Data structures
23
+ # ---------------------------------------------------------------------------
24
+
25
+ @dataclass
26
+ class DatabaseSchema:
27
+ """A database domain with its own schema, seed data, and description."""
28
+ id: str
29
+ name: str
30
+ schema_sql: str
31
+ seed_sql: str
32
+ description: str
33
+
34
+
35
+ @dataclass
36
+ class Task:
37
+ """A single SQL query task with ground truth."""
38
+ id: str
39
+ difficulty: str # "easy", "medium", "hard"
40
+ schema_id: str # Which database schema to use
41
+ question: str
42
+ ground_truth_query: str
43
+ expected_columns: List[str]
44
+ description: str = ""
45
+ hints: List[str] = field(default_factory=list)
46
+ max_steps: int = 5
47
+ challenge_type: str = "" # e.g., "ambiguous", "temporal", "efficiency"
48
+
49
+
50
+ # ---------------------------------------------------------------------------
51
+ # Schema 1: Company Analytics
52
+ # ---------------------------------------------------------------------------
53
+
54
+ COMPANY_SCHEMA_SQL = """
55
+ CREATE TABLE departments (
56
+ id INTEGER PRIMARY KEY,
57
+ name TEXT NOT NULL,
58
+ budget REAL NOT NULL,
59
+ location TEXT NOT NULL
60
+ );
61
+
62
+ CREATE TABLE employees (
63
+ id INTEGER PRIMARY KEY,
64
+ name TEXT NOT NULL,
65
+ email TEXT NOT NULL,
66
+ department_id INTEGER NOT NULL,
67
+ salary REAL NOT NULL,
68
+ hire_date TEXT NOT NULL,
69
+ manager_id INTEGER,
70
+ is_active INTEGER DEFAULT 1,
71
+ FOREIGN KEY (department_id) REFERENCES departments(id),
72
+ FOREIGN KEY (manager_id) REFERENCES employees(id)
73
+ );
74
+
75
+ CREATE TABLE products (
76
+ id INTEGER PRIMARY KEY,
77
+ name TEXT NOT NULL,
78
+ category TEXT NOT NULL,
79
+ price REAL NOT NULL,
80
+ stock INTEGER NOT NULL DEFAULT 0,
81
+ created_date TEXT NOT NULL
82
+ );
83
+
84
+ CREATE TABLE customers (
85
+ id INTEGER PRIMARY KEY,
86
+ name TEXT NOT NULL,
87
+ email TEXT NOT NULL,
88
+ city TEXT NOT NULL,
89
+ country TEXT NOT NULL,
90
+ join_date TEXT NOT NULL,
91
+ tier TEXT NOT NULL DEFAULT 'standard'
92
+ );
93
+
94
+ CREATE TABLE orders (
95
+ id INTEGER PRIMARY KEY,
96
+ customer_id INTEGER NOT NULL,
97
+ product_id INTEGER NOT NULL,
98
+ quantity INTEGER NOT NULL,
99
+ total_price REAL NOT NULL,
100
+ order_date TEXT NOT NULL,
101
+ status TEXT NOT NULL DEFAULT 'pending',
102
+ FOREIGN KEY (customer_id) REFERENCES customers(id),
103
+ FOREIGN KEY (product_id) REFERENCES products(id)
104
+ );
105
+
106
+ CREATE TABLE reviews (
107
+ id INTEGER PRIMARY KEY,
108
+ product_id INTEGER NOT NULL,
109
+ customer_id INTEGER NOT NULL,
110
+ rating INTEGER NOT NULL CHECK (rating BETWEEN 1 AND 5),
111
+ review_text TEXT,
112
+ review_date TEXT NOT NULL,
113
+ FOREIGN KEY (product_id) REFERENCES products(id),
114
+ FOREIGN KEY (customer_id) REFERENCES customers(id)
115
+ );
116
+ """
117
+
118
+ COMPANY_SEED_SQL = """
119
+ INSERT INTO departments VALUES (1, 'Engineering', 500000.00, 'San Francisco');
120
+ INSERT INTO departments VALUES (2, 'Marketing', 250000.00, 'New York');
121
+ INSERT INTO departments VALUES (3, 'Sales', 300000.00, 'Chicago');
122
+ INSERT INTO departments VALUES (4, 'Human Resources', 150000.00, 'San Francisco');
123
+ INSERT INTO departments VALUES (5, 'Finance', 200000.00, 'New York');
124
+
125
+ INSERT INTO employees VALUES (1, 'Alice Chen', 'alice@company.com', 1, 145000, '2020-03-15', NULL, 1);
126
+ INSERT INTO employees VALUES (2, 'Bob Smith', 'bob@company.com', 1, 130000, '2021-06-01', 1, 1);
127
+ INSERT INTO employees VALUES (3, 'Carol Davis', 'carol@company.com', 2, 95000, '2019-11-20', NULL, 1);
128
+ INSERT INTO employees VALUES (4, 'David Wilson', 'david@company.com', 1, 125000, '2022-01-10', 1, 1);
129
+ INSERT INTO employees VALUES (5, 'Eva Martinez', 'eva@company.com', 3, 110000, '2020-08-05', NULL, 1);
130
+ INSERT INTO employees VALUES (6, 'Frank Brown', 'frank@company.com', 3, 98000, '2021-04-22', 5, 1);
131
+ INSERT INTO employees VALUES (7, 'Grace Lee', 'grace@company.com', 2, 105000, '2022-07-14', 3, 1);
132
+ INSERT INTO employees VALUES (8, 'Henry Taylor', 'henry@company.com', 4, 88000, '2023-02-28', NULL, 1);
133
+ INSERT INTO employees VALUES (9, 'Ivy Nguyen', 'ivy@company.com', 5, 115000, '2020-12-01', NULL, 1);
134
+ INSERT INTO employees VALUES (10, 'Jack Anderson', 'jack@company.com', 1, 140000, '2019-09-10', 1, 1);
135
+ INSERT INTO employees VALUES (11, 'Karen White', 'karen@company.com', 5, 105000, '2021-10-15', 9, 1);
136
+ INSERT INTO employees VALUES (12, 'Leo Garcia', 'leo@company.com', 3, 92000, '2023-05-20', 5, 0);
137
+ INSERT INTO employees VALUES (13, 'Mia Johnson', 'mia@company.com', 2, 88000, '2022-11-08', 3, 1);
138
+ INSERT INTO employees VALUES (14, 'Nathan Park', 'nathan@company.com', 1, 155000, '2018-06-25', NULL, 1);
139
+ INSERT INTO employees VALUES (15, 'Olivia Kim', 'olivia@company.com', 4, 82000, '2023-08-12', 8, 1);
140
+
141
+ INSERT INTO products VALUES (1, 'Laptop Pro 15', 'Electronics', 1299.99, 45, '2023-01-15');
142
+ INSERT INTO products VALUES (2, 'Wireless Mouse', 'Electronics', 29.99, 200, '2023-02-20');
143
+ INSERT INTO products VALUES (3, 'Standing Desk', 'Furniture', 549.99, 30, '2023-03-10');
144
+ INSERT INTO products VALUES (4, 'Noise-Canceling Headphones', 'Electronics', 199.99, 75, '2023-01-25');
145
+ INSERT INTO products VALUES (5, 'Ergonomic Chair', 'Furniture', 399.99, 25, '2023-04-05');
146
+ INSERT INTO products VALUES (6, 'USB-C Hub', 'Electronics', 49.99, 150, '2023-05-12');
147
+ INSERT INTO products VALUES (7, 'Monitor 27inch', 'Electronics', 449.99, 60, '2023-02-14');
148
+ INSERT INTO products VALUES (8, 'Keyboard Mechanical', 'Electronics', 89.99, 100, '2023-06-01');
149
+ INSERT INTO products VALUES (9, 'Desk Lamp', 'Furniture', 39.99, 80, '2023-03-22');
150
+ INSERT INTO products VALUES (10, 'Webcam HD', 'Electronics', 79.99, 90, '2023-07-08');
151
+
152
+ INSERT INTO customers VALUES (1, 'TechCorp Inc', 'orders@techcorp.com', 'San Francisco', 'USA', '2023-01-10', 'premium');
153
+ INSERT INTO customers VALUES (2, 'DataFlow Ltd', 'buy@dataflow.io', 'London', 'UK', '2023-02-15', 'standard');
154
+ INSERT INTO customers VALUES (3, 'StartupXYZ', 'procurement@startupxyz.com', 'Berlin', 'Germany', '2023-03-20', 'standard');
155
+ INSERT INTO customers VALUES (4, 'MegaCorp', 'orders@megacorp.com', 'Tokyo', 'Japan', '2023-01-05', 'premium');
156
+ INSERT INTO customers VALUES (5, 'CloudNine', 'buy@cloudnine.com', 'New York', 'USA', '2023-04-12', 'premium');
157
+ INSERT INTO customers VALUES (6, 'InnovateLab', 'purchasing@innovatelab.com', 'Mumbai', 'India', '2023-05-08', 'standard');
158
+ INSERT INTO customers VALUES (7, 'AlphaWorks', 'orders@alphaworks.net', 'Sydney', 'Australia', '2023-02-28', 'standard');
159
+ INSERT INTO customers VALUES (8, 'BetaSoft', 'buy@betasoft.co', 'Toronto', 'Canada', '2023-06-15', 'premium');
160
+
161
+ INSERT INTO orders VALUES (1, 1, 1, 5, 6499.95, '2024-01-15', 'completed');
162
+ INSERT INTO orders VALUES (2, 1, 4, 10, 1999.90, '2024-01-20', 'completed');
163
+ INSERT INTO orders VALUES (3, 2, 2, 20, 599.80, '2024-02-05', 'completed');
164
+ INSERT INTO orders VALUES (4, 3, 3, 3, 1649.97, '2024-02-10', 'completed');
165
+ INSERT INTO orders VALUES (5, 4, 1, 8, 10399.92, '2024-02-15', 'completed');
166
+ INSERT INTO orders VALUES (6, 5, 7, 4, 1799.96, '2024-03-01', 'completed');
167
+ INSERT INTO orders VALUES (7, 1, 6, 15, 749.85, '2024-03-10', 'completed');
168
+ INSERT INTO orders VALUES (8, 6, 5, 2, 799.98, '2024-03-15', 'shipped');
169
+ INSERT INTO orders VALUES (9, 2, 8, 5, 449.95, '2024-03-20', 'shipped');
170
+ INSERT INTO orders VALUES (10, 4, 2, 50, 1499.50, '2024-04-01', 'completed');
171
+ INSERT INTO orders VALUES (11, 7, 9, 10, 399.90, '2024-04-05', 'completed');
172
+ INSERT INTO orders VALUES (12, 3, 10, 6, 479.94, '2024-04-10', 'shipped');
173
+ INSERT INTO orders VALUES (13, 8, 1, 3, 3899.97, '2024-04-15', 'pending');
174
+ INSERT INTO orders VALUES (14, 5, 5, 5, 1999.95, '2024-04-20', 'completed');
175
+ INSERT INTO orders VALUES (15, 6, 8, 8, 719.92, '2024-05-01', 'completed');
176
+ INSERT INTO orders VALUES (16, 1, 3, 2, 1099.98, '2024-05-05', 'completed');
177
+ INSERT INTO orders VALUES (17, 4, 6, 25, 1249.75, '2024-05-10', 'shipped');
178
+ INSERT INTO orders VALUES (18, 7, 4, 3, 599.97, '2024-05-15', 'completed');
179
+ INSERT INTO orders VALUES (19, 2, 7, 2, 899.98, '2024-05-20', 'pending');
180
+ INSERT INTO orders VALUES (20, 8, 10, 4, 319.96, '2024-05-25', 'completed');
181
+ INSERT INTO orders VALUES (21, 5, 2, 30, 899.70, '2024-06-01', 'completed');
182
+ INSERT INTO orders VALUES (22, 3, 1, 2, 2599.98, '2024-06-05', 'completed');
183
+ INSERT INTO orders VALUES (23, 6, 3, 1, 549.99, '2024-06-10', 'shipped');
184
+ INSERT INTO orders VALUES (24, 4, 8, 12, 1079.88, '2024-06-15', 'completed');
185
+
186
+ INSERT INTO reviews VALUES (1, 1, 1, 5, 'Excellent laptop, very fast!', '2024-02-01');
187
+ INSERT INTO reviews VALUES (2, 1, 4, 4, 'Good performance but heavy', '2024-03-01');
188
+ INSERT INTO reviews VALUES (3, 2, 2, 4, 'Good mouse, comfortable grip', '2024-02-20');
189
+ INSERT INTO reviews VALUES (4, 3, 3, 5, 'Best standing desk ever!', '2024-03-01');
190
+ INSERT INTO reviews VALUES (5, 4, 1, 5, 'Amazing noise cancellation', '2024-02-10');
191
+ INSERT INTO reviews VALUES (6, 5, 6, 3, 'Decent chair but could be better', '2024-04-01');
192
+ INSERT INTO reviews VALUES (7, 7, 5, 4, 'Great display quality', '2024-03-15');
193
+ INSERT INTO reviews VALUES (8, 8, 2, 5, 'Perfect mechanical keyboard', '2024-04-05');
194
+ INSERT INTO reviews VALUES (9, 6, 1, 4, 'Works well with all my devices', '2024-03-20');
195
+ INSERT INTO reviews VALUES (10, 10, 7, 3, 'Average webcam quality', '2024-05-01');
196
+ INSERT INTO reviews VALUES (11, 1, 8, 5, 'Worth every penny', '2024-05-10');
197
+ INSERT INTO reviews VALUES (12, 4, 5, 4, 'Great sound quality', '2024-05-20');
198
+ INSERT INTO reviews VALUES (13, 9, 3, 4, 'Nice desk lamp, adjustable', '2024-04-15');
199
+ INSERT INTO reviews VALUES (14, 5, 8, 2, 'Armrests broke after 2 months', '2024-06-01');
200
+ INSERT INTO reviews VALUES (15, 2, 6, 4, 'Reliable and affordable', '2024-06-10');
201
+ """
202
+
203
+ COMPANY_DESCRIPTION = """DATABASE SCHEMA β€” Company Analytics
204
+ =======================================
205
+
206
+ TABLE: departments
207
+ - id (INTEGER, PK) | name (TEXT) | budget (REAL) | location (TEXT)
208
+
209
+ TABLE: employees
210
+ - id (INTEGER, PK) | name (TEXT) | email (TEXT) | department_id (FK→departments)
211
+ - salary (REAL) | hire_date (TEXT, YYYY-MM-DD) | manager_id (FK→employees, nullable)
212
+ - is_active (INTEGER, 1=active/0=inactive)
213
+
214
+ TABLE: products
215
+ - id (INTEGER, PK) | name (TEXT) | category (TEXT, 'Electronics'/'Furniture')
216
+ - price (REAL) | stock (INTEGER) | created_date (TEXT)
217
+
218
+ TABLE: customers
219
+ - id (INTEGER, PK) | name (TEXT) | email (TEXT) | city (TEXT) | country (TEXT)
220
+ - join_date (TEXT) | tier (TEXT, 'standard'/'premium')
221
+
222
+ TABLE: orders
223
+ - id (INTEGER, PK) | customer_id (FK→customers) | product_id (FK→products)
224
+ - quantity (INTEGER) | total_price (REAL) | order_date (TEXT)
225
+ - status (TEXT, 'pending'/'shipped'/'completed')
226
+
227
+ TABLE: reviews
228
+ - id (INTEGER, PK) | product_id (FK→products) | customer_id (FK→customers)
229
+ - rating (INTEGER, 1-5) | review_text (TEXT, nullable) | review_date (TEXT)
230
+
231
+ RELATIONSHIPS:
232
+ employees.department_id β†’ departments.id
233
+ employees.manager_id β†’ employees.id (self-referencing, hierarchical)
234
+ orders.customer_id β†’ customers.id
235
+ orders.product_id β†’ products.id
236
+ reviews.product_id β†’ products.id
237
+ reviews.customer_id β†’ customers.id
238
+ """
239
+
240
+
241
+ # ---------------------------------------------------------------------------
242
+ # Schema 2: Hospital Management
243
+ # ---------------------------------------------------------------------------
244
+
245
+ HOSPITAL_SCHEMA_SQL = """
246
+ CREATE TABLE wards (
247
+ id INTEGER PRIMARY KEY,
248
+ name TEXT NOT NULL,
249
+ capacity INTEGER NOT NULL,
250
+ floor INTEGER NOT NULL
251
+ );
252
+
253
+ CREATE TABLE doctors (
254
+ id INTEGER PRIMARY KEY,
255
+ name TEXT NOT NULL,
256
+ specialization TEXT NOT NULL,
257
+ ward_id INTEGER NOT NULL,
258
+ experience_years INTEGER NOT NULL,
259
+ is_available INTEGER DEFAULT 1,
260
+ FOREIGN KEY (ward_id) REFERENCES wards(id)
261
+ );
262
+
263
+ CREATE TABLE patients (
264
+ id INTEGER PRIMARY KEY,
265
+ name TEXT NOT NULL,
266
+ age INTEGER NOT NULL,
267
+ gender TEXT NOT NULL,
268
+ ward_id INTEGER,
269
+ admission_date TEXT NOT NULL,
270
+ discharge_date TEXT,
271
+ diagnosis TEXT NOT NULL,
272
+ insurance_type TEXT NOT NULL DEFAULT 'none',
273
+ FOREIGN KEY (ward_id) REFERENCES wards(id)
274
+ );
275
+
276
+ CREATE TABLE appointments (
277
+ id INTEGER PRIMARY KEY,
278
+ patient_id INTEGER NOT NULL,
279
+ doctor_id INTEGER NOT NULL,
280
+ appointment_date TEXT NOT NULL,
281
+ appointment_type TEXT NOT NULL,
282
+ status TEXT NOT NULL DEFAULT 'scheduled',
283
+ notes TEXT,
284
+ FOREIGN KEY (patient_id) REFERENCES patients(id),
285
+ FOREIGN KEY (doctor_id) REFERENCES doctors(id)
286
+ );
287
+
288
+ CREATE TABLE medications (
289
+ id INTEGER PRIMARY KEY,
290
+ name TEXT NOT NULL,
291
+ category TEXT NOT NULL,
292
+ unit_cost REAL NOT NULL
293
+ );
294
+
295
+ CREATE TABLE prescriptions (
296
+ id INTEGER PRIMARY KEY,
297
+ patient_id INTEGER NOT NULL,
298
+ doctor_id INTEGER NOT NULL,
299
+ medication_id INTEGER NOT NULL,
300
+ dosage TEXT NOT NULL,
301
+ frequency TEXT NOT NULL,
302
+ start_date TEXT NOT NULL,
303
+ end_date TEXT,
304
+ FOREIGN KEY (patient_id) REFERENCES patients(id),
305
+ FOREIGN KEY (doctor_id) REFERENCES doctors(id),
306
+ FOREIGN KEY (medication_id) REFERENCES medications(id)
307
+ );
308
+ """
309
+
310
+ HOSPITAL_SEED_SQL = """
311
+ INSERT INTO wards VALUES (1, 'Cardiology', 30, 2);
312
+ INSERT INTO wards VALUES (2, 'Neurology', 20, 3);
313
+ INSERT INTO wards VALUES (3, 'Orthopedics', 25, 1);
314
+ INSERT INTO wards VALUES (4, 'Pediatrics', 35, 1);
315
+ INSERT INTO wards VALUES (5, 'Emergency', 40, 0);
316
+
317
+ INSERT INTO doctors VALUES (1, 'Dr. Sarah Patel', 'Cardiologist', 1, 15, 1);
318
+ INSERT INTO doctors VALUES (2, 'Dr. James Liu', 'Neurologist', 2, 12, 1);
319
+ INSERT INTO doctors VALUES (3, 'Dr. Maria Garcia', 'Orthopedic Surgeon', 3, 20, 1);
320
+ INSERT INTO doctors VALUES (4, 'Dr. Robert Kim', 'Pediatrician', 4, 8, 1);
321
+ INSERT INTO doctors VALUES (5, 'Dr. Emily Chen', 'Emergency Medicine', 5, 10, 1);
322
+ INSERT INTO doctors VALUES (6, 'Dr. Michael Brown', 'Cardiologist', 1, 5, 0);
323
+ INSERT INTO doctors VALUES (7, 'Dr. Lisa Wong', 'Neurologist', 2, 18, 1);
324
+ INSERT INTO doctors VALUES (8, 'Dr. David Okafor', 'Pediatrician', 4, 3, 1);
325
+
326
+ INSERT INTO patients VALUES (1, 'John Doe', 65, 'M', 1, '2024-01-10', '2024-01-20', 'Chest Pain', 'private');
327
+ INSERT INTO patients VALUES (2, 'Jane Smith', 45, 'F', 2, '2024-01-15', '2024-02-01', 'Migraine', 'government');
328
+ INSERT INTO patients VALUES (3, 'Bob Johnson', 72, 'M', 1, '2024-02-01', NULL, 'Heart Failure', 'private');
329
+ INSERT INTO patients VALUES (4, 'Alice Williams', 8, 'F', 4, '2024-02-10', '2024-02-15', 'Flu', 'private');
330
+ INSERT INTO patients VALUES (5, 'Charlie Brown', 55, 'M', 3, '2024-02-20', '2024-03-15', 'Fracture', 'government');
331
+ INSERT INTO patients VALUES (6, 'Diana Ross', 38, 'F', 5, '2024-03-01', '2024-03-02', 'Laceration', 'none');
332
+ INSERT INTO patients VALUES (7, 'Edward Lee', 80, 'M', 1, '2024-03-10', NULL, 'Arrhythmia', 'government');
333
+ INSERT INTO patients VALUES (8, 'Fiona Clark', 12, 'F', 4, '2024-03-15', '2024-03-18', 'Asthma', 'private');
334
+ INSERT INTO patients VALUES (9, 'George Martin', 60, 'M', 2, '2024-03-20', '2024-04-05', 'Stroke', 'private');
335
+ INSERT INTO patients VALUES (10, 'Helen Park', 50, 'F', 3, '2024-04-01', NULL, 'Joint Replacement', 'government');
336
+ INSERT INTO patients VALUES (11, 'Ivan Petrov', 70, 'M', 5, '2024-04-10', '2024-04-11', 'Chest Pain', 'none');
337
+ INSERT INTO patients VALUES (12, 'Julia Santos', 5, 'F', 4, '2024-04-15', '2024-04-17', 'Ear Infection', 'private');
338
+
339
+ INSERT INTO appointments VALUES (1, 1, 1, '2024-01-12', 'Follow-up', 'completed', 'ECG normal');
340
+ INSERT INTO appointments VALUES (2, 2, 2, '2024-01-20', 'Consultation', 'completed', 'MRI scheduled');
341
+ INSERT INTO appointments VALUES (3, 3, 1, '2024-02-05', 'Initial', 'completed', 'Admitted to ICU');
342
+ INSERT INTO appointments VALUES (4, 4, 4, '2024-02-12', 'Follow-up', 'completed', 'Recovering well');
343
+ INSERT INTO appointments VALUES (5, 5, 3, '2024-02-25', 'Surgery', 'completed', 'Pin insertion');
344
+ INSERT INTO appointments VALUES (6, 6, 5, '2024-03-01', 'Emergency', 'completed', 'Stitches applied');
345
+ INSERT INTO appointments VALUES (7, 7, 1, '2024-03-12', 'Initial', 'completed', 'Pacemaker evaluation');
346
+ INSERT INTO appointments VALUES (8, 9, 7, '2024-03-22', 'Initial', 'completed', 'CT scan ordered');
347
+ INSERT INTO appointments VALUES (9, 3, 6, '2024-03-15', 'Follow-up', 'cancelled', NULL);
348
+ INSERT INTO appointments VALUES (10, 10, 3, '2024-04-03', 'Surgery', 'scheduled', 'Pre-op clearance needed');
349
+ INSERT INTO appointments VALUES (11, 11, 5, '2024-04-10', 'Emergency', 'completed', 'Ruled out MI');
350
+ INSERT INTO appointments VALUES (12, 1, 1, '2024-04-15', 'Follow-up', 'scheduled', NULL);
351
+ INSERT INTO appointments VALUES (13, 8, 4, '2024-03-16', 'Follow-up', 'completed', 'Inhaler prescribed');
352
+ INSERT INTO appointments VALUES (14, 9, 2, '2024-04-01', 'Follow-up', 'completed', 'Rehab started');
353
+ INSERT INTO appointments VALUES (15, 1, 1, '2024-05-10', 'Follow-up', 'completed', 'Stable condition');
354
+ INSERT INTO appointments VALUES (16, 3, 1, '2024-04-20', 'Follow-up', 'completed', 'Medication adjusted');
355
+ INSERT INTO appointments VALUES (17, 9, 7, '2024-04-25', 'Follow-up', 'completed', 'Good progress');
356
+
357
+ INSERT INTO medications VALUES (1, 'Aspirin', 'Blood Thinner', 0.50);
358
+ INSERT INTO medications VALUES (2, 'Metoprolol', 'Beta Blocker', 1.20);
359
+ INSERT INTO medications VALUES (3, 'Sumatriptan', 'Migraine Relief', 8.50);
360
+ INSERT INTO medications VALUES (4, 'Amoxicillin', 'Antibiotic', 0.80);
361
+ INSERT INTO medications VALUES (5, 'Ibuprofen', 'Pain Relief', 0.30);
362
+ INSERT INTO medications VALUES (6, 'Albuterol', 'Bronchodilator', 15.00);
363
+ INSERT INTO medications VALUES (7, 'Warfarin', 'Anticoagulant', 2.00);
364
+ INSERT INTO medications VALUES (8, 'Lisinopril', 'ACE Inhibitor', 0.90);
365
+
366
+ INSERT INTO prescriptions VALUES (1, 1, 1, 1, '81mg', 'daily', '2024-01-12', NULL);
367
+ INSERT INTO prescriptions VALUES (2, 1, 1, 2, '50mg', 'twice daily', '2024-01-12', '2024-03-12');
368
+ INSERT INTO prescriptions VALUES (3, 2, 2, 3, '50mg', 'as needed', '2024-01-20', NULL);
369
+ INSERT INTO prescriptions VALUES (4, 3, 1, 7, '5mg', 'daily', '2024-02-05', NULL);
370
+ INSERT INTO prescriptions VALUES (5, 3, 1, 8, '10mg', 'daily', '2024-02-05', NULL);
371
+ INSERT INTO prescriptions VALUES (6, 5, 3, 5, '400mg', 'three times daily', '2024-02-25', '2024-03-25');
372
+ INSERT INTO prescriptions VALUES (7, 8, 4, 6, '2 puffs', 'as needed', '2024-03-16', NULL);
373
+ INSERT INTO prescriptions VALUES (8, 7, 1, 2, '25mg', 'daily', '2024-03-12', NULL);
374
+ INSERT INTO prescriptions VALUES (9, 9, 7, 1, '325mg', 'daily', '2024-03-22', NULL);
375
+ INSERT INTO prescriptions VALUES (10, 12, 4, 4, '250mg', 'three times daily', '2024-04-15', '2024-04-25');
376
+ INSERT INTO prescriptions VALUES (11, 11, 1, 1, '81mg', 'daily', '2024-04-10', NULL);
377
+ INSERT INTO prescriptions VALUES (12, 8, 5, 5, '200mg', 'as needed', '2024-03-17', '2024-03-20');
378
+ INSERT INTO prescriptions VALUES (13, 10, 7, 5, '400mg', 'twice daily', '2024-04-05', NULL);
379
+ """
380
+
381
+ HOSPITAL_DESCRIPTION = """DATABASE SCHEMA β€” Hospital Management
382
+ =======================================
383
+
384
+ TABLE: wards
385
+ - id (INTEGER, PK) | name (TEXT) | capacity (INTEGER) | floor (INTEGER)
386
+
387
+ TABLE: doctors
388
+ - id (INTEGER, PK) | name (TEXT) | specialization (TEXT)
389
+ - ward_id (FK→wards) | experience_years (INTEGER) | is_available (INTEGER)
390
+
391
+ TABLE: patients
392
+ - id (INTEGER, PK) | name (TEXT) | age (INTEGER) | gender (TEXT, 'M'/'F')
393
+ - ward_id (FK→wards, nullable) | admission_date (TEXT) | discharge_date (TEXT, nullable)
394
+ - diagnosis (TEXT) | insurance_type (TEXT, 'private'/'government'/'none')
395
+
396
+ TABLE: appointments
397
+ - id (INTEGER, PK) | patient_id (FK→patients) | doctor_id (FK→doctors)
398
+ - appointment_date (TEXT) | appointment_type (TEXT)
399
+ - status (TEXT, 'scheduled'/'completed'/'cancelled') | notes (TEXT, nullable)
400
+
401
+ TABLE: medications
402
+ - id (INTEGER, PK) | name (TEXT) | category (TEXT) | unit_cost (REAL)
403
+
404
+ TABLE: prescriptions
405
+ - id (INTEGER, PK) | patient_id (FK→patients) | doctor_id (FK→doctors)
406
+ - medication_id (FK→medications) | dosage (TEXT) | frequency (TEXT)
407
+ - start_date (TEXT) | end_date (TEXT, nullable β€” NULL means ongoing)
408
+
409
+ RELATIONSHIPS:
410
+ doctors.ward_id β†’ wards.id
411
+ patients.ward_id β†’ wards.id (nullable β€” outpatients have no ward)
412
+ appointments.patient_id β†’ patients.id
413
+ appointments.doctor_id β†’ doctors.id
414
+ prescriptions.patient_id β†’ patients.id
415
+ prescriptions.doctor_id β†’ doctors.id
416
+ prescriptions.medication_id β†’ medications.id
417
+
418
+ NOTE: discharge_date = NULL means patient is still admitted.
419
+ end_date = NULL in prescriptions means medication is ongoing.
420
+ """
421
+
422
+
423
+ # ---------------------------------------------------------------------------
424
+ # Schema 3: E-Commerce Platform
425
+ # ---------------------------------------------------------------------------
426
+
427
+ ECOMMERCE_SCHEMA_SQL = """
428
+ CREATE TABLE sellers (
429
+ id INTEGER PRIMARY KEY,
430
+ name TEXT NOT NULL,
431
+ rating REAL NOT NULL,
432
+ city TEXT NOT NULL,
433
+ joined_date TEXT NOT NULL
434
+ );
435
+
436
+ CREATE TABLE categories (
437
+ id INTEGER PRIMARY KEY,
438
+ name TEXT NOT NULL,
439
+ parent_category TEXT
440
+ );
441
+
442
+ CREATE TABLE products (
443
+ id INTEGER PRIMARY KEY,
444
+ name TEXT NOT NULL,
445
+ category_id INTEGER NOT NULL,
446
+ seller_id INTEGER NOT NULL,
447
+ price REAL NOT NULL,
448
+ stock INTEGER NOT NULL DEFAULT 0,
449
+ listed_date TEXT NOT NULL,
450
+ FOREIGN KEY (category_id) REFERENCES categories(id),
451
+ FOREIGN KEY (seller_id) REFERENCES sellers(id)
452
+ );
453
+
454
+ CREATE TABLE users (
455
+ id INTEGER PRIMARY KEY,
456
+ name TEXT NOT NULL,
457
+ email TEXT NOT NULL,
458
+ city TEXT NOT NULL,
459
+ signup_date TEXT NOT NULL,
460
+ is_premium INTEGER DEFAULT 0
461
+ );
462
+
463
+ CREATE TABLE orders (
464
+ id INTEGER PRIMARY KEY,
465
+ user_id INTEGER NOT NULL,
466
+ order_date TEXT NOT NULL,
467
+ total_amount REAL NOT NULL,
468
+ status TEXT NOT NULL DEFAULT 'pending',
469
+ payment_method TEXT NOT NULL,
470
+ FOREIGN KEY (user_id) REFERENCES users(id)
471
+ );
472
+
473
+ CREATE TABLE order_items (
474
+ id INTEGER PRIMARY KEY,
475
+ order_id INTEGER NOT NULL,
476
+ product_id INTEGER NOT NULL,
477
+ quantity INTEGER NOT NULL,
478
+ unit_price REAL NOT NULL,
479
+ FOREIGN KEY (order_id) REFERENCES orders(id),
480
+ FOREIGN KEY (product_id) REFERENCES products(id)
481
+ );
482
+
483
+ CREATE TABLE returns (
484
+ id INTEGER PRIMARY KEY,
485
+ order_item_id INTEGER NOT NULL,
486
+ reason TEXT NOT NULL,
487
+ return_date TEXT NOT NULL,
488
+ refund_amount REAL NOT NULL,
489
+ status TEXT NOT NULL DEFAULT 'pending',
490
+ FOREIGN KEY (order_item_id) REFERENCES order_items(id)
491
+ );
492
+ """
493
+
494
+ ECOMMERCE_SEED_SQL = """
495
+ INSERT INTO sellers VALUES (1, 'TechWorld', 4.5, 'Shenzhen', '2022-01-10');
496
+ INSERT INTO sellers VALUES (2, 'HomeStyle', 4.2, 'Mumbai', '2022-03-15');
497
+ INSERT INTO sellers VALUES (3, 'FashionHub', 3.8, 'Milan', '2022-06-20');
498
+ INSERT INTO sellers VALUES (4, 'BookBarn', 4.7, 'New York', '2021-11-01');
499
+ INSERT INTO sellers VALUES (5, 'SportZone', 4.0, 'Tokyo', '2023-02-14');
500
+
501
+ INSERT INTO categories VALUES (1, 'Electronics', NULL);
502
+ INSERT INTO categories VALUES (2, 'Smartphones', 'Electronics');
503
+ INSERT INTO categories VALUES (3, 'Laptops', 'Electronics');
504
+ INSERT INTO categories VALUES (4, 'Home & Kitchen', NULL);
505
+ INSERT INTO categories VALUES (5, 'Furniture', 'Home & Kitchen');
506
+ INSERT INTO categories VALUES (6, 'Appliances', 'Home & Kitchen');
507
+ INSERT INTO categories VALUES (7, 'Books', NULL);
508
+ INSERT INTO categories VALUES (8, 'Fiction', 'Books');
509
+ INSERT INTO categories VALUES (9, 'Non-Fiction', 'Books');
510
+ INSERT INTO categories VALUES (10, 'Sports', NULL);
511
+
512
+ INSERT INTO products VALUES (1, 'Galaxy S24', 2, 1, 899.99, 120, '2024-01-15');
513
+ INSERT INTO products VALUES (2, 'MacBook Air M3', 3, 1, 1199.99, 45, '2024-01-20');
514
+ INSERT INTO products VALUES (3, 'Blender Pro', 6, 2, 79.99, 200, '2024-02-01');
515
+ INSERT INTO products VALUES (4, 'Oak Dining Table', 5, 2, 599.99, 15, '2024-02-10');
516
+ INSERT INTO products VALUES (5, 'Running Shoes X1', 10, 5, 129.99, 300, '2024-01-05');
517
+ INSERT INTO products VALUES (6, 'Python Crash Course', 9, 4, 39.99, 500, '2024-03-01');
518
+ INSERT INTO products VALUES (7, 'The Great Gatsby', 8, 4, 12.99, 350, '2024-01-10');
519
+ INSERT INTO products VALUES (8, 'Yoga Mat Premium', 10, 5, 49.99, 180, '2024-02-20');
520
+ INSERT INTO products VALUES (9, 'Coffee Maker Deluxe', 6, 2, 149.99, 90, '2024-03-05');
521
+ INSERT INTO products VALUES (10, 'Pixel 8 Pro', 2, 1, 999.99, 80, '2024-03-10');
522
+ INSERT INTO products VALUES (11, 'Standing Desk Frame', 5, 2, 349.99, 40, '2024-03-15');
523
+ INSERT INTO products VALUES (12, 'Dune: Part One', 8, 4, 14.99, 200, '2024-02-01');
524
+
525
+ INSERT INTO users VALUES (1, 'Priya Sharma', 'priya@email.com', 'Mumbai', '2023-06-15', 1);
526
+ INSERT INTO users VALUES (2, 'Tom Baker', 'tom@email.com', 'London', '2023-08-20', 0);
527
+ INSERT INTO users VALUES (3, 'Yuki Tanaka', 'yuki@email.com', 'Tokyo', '2023-09-01', 1);
528
+ INSERT INTO users VALUES (4, 'Carlos Mendez', 'carlos@email.com', 'Mexico City', '2023-11-10', 0);
529
+ INSERT INTO users VALUES (5, 'Anna Kowalski', 'anna@email.com', 'Warsaw', '2024-01-05', 1);
530
+ INSERT INTO users VALUES (6, 'Wei Zhang', 'wei@email.com', 'Shanghai', '2024-02-14', 0);
531
+
532
+ INSERT INTO orders VALUES (1, 1, '2024-01-20', 899.99, 'delivered', 'credit_card');
533
+ INSERT INTO orders VALUES (2, 1, '2024-02-15', 1279.98, 'delivered', 'credit_card');
534
+ INSERT INTO orders VALUES (3, 2, '2024-02-01', 79.99, 'delivered', 'paypal');
535
+ INSERT INTO orders VALUES (4, 3, '2024-02-20', 1329.98, 'delivered', 'credit_card');
536
+ INSERT INTO orders VALUES (5, 4, '2024-03-01', 52.98, 'delivered', 'debit_card');
537
+ INSERT INTO orders VALUES (6, 5, '2024-03-10', 899.99, 'delivered', 'credit_card');
538
+ INSERT INTO orders VALUES (7, 1, '2024-03-15', 199.98, 'shipped', 'credit_card');
539
+ INSERT INTO orders VALUES (8, 2, '2024-03-20', 599.99, 'delivered', 'paypal');
540
+ INSERT INTO orders VALUES (9, 3, '2024-04-01', 179.98, 'delivered', 'credit_card');
541
+ INSERT INTO orders VALUES (10, 6, '2024-04-05', 1199.99, 'pending', 'credit_card');
542
+ INSERT INTO orders VALUES (11, 4, '2024-04-10', 129.99, 'delivered', 'debit_card');
543
+ INSERT INTO orders VALUES (12, 5, '2024-04-15', 349.99, 'cancelled', 'credit_card');
544
+
545
+ INSERT INTO order_items VALUES (1, 1, 1, 1, 899.99);
546
+ INSERT INTO order_items VALUES (2, 2, 2, 1, 1199.99);
547
+ INSERT INTO order_items VALUES (3, 2, 3, 1, 79.99);
548
+ INSERT INTO order_items VALUES (4, 3, 3, 1, 79.99);
549
+ INSERT INTO order_items VALUES (5, 4, 2, 1, 1199.99);
550
+ INSERT INTO order_items VALUES (6, 4, 5, 1, 129.99);
551
+ INSERT INTO order_items VALUES (7, 5, 6, 1, 39.99);
552
+ INSERT INTO order_items VALUES (8, 5, 7, 1, 12.99);
553
+ INSERT INTO order_items VALUES (9, 6, 1, 1, 899.99);
554
+ INSERT INTO order_items VALUES (10, 7, 8, 2, 49.99);
555
+ INSERT INTO order_items VALUES (11, 7, 9, 1, 149.99);
556
+ INSERT INTO order_items VALUES (12, 8, 4, 1, 599.99);
557
+ INSERT INTO order_items VALUES (13, 9, 5, 1, 129.99);
558
+ INSERT INTO order_items VALUES (14, 9, 8, 1, 49.99);
559
+ INSERT INTO order_items VALUES (15, 10, 2, 1, 1199.99);
560
+ INSERT INTO order_items VALUES (16, 11, 5, 1, 129.99);
561
+ INSERT INTO order_items VALUES (17, 12, 11, 1, 349.99);
562
+
563
+ INSERT INTO returns VALUES (1, 4, 'Defective product', '2024-02-10', 79.99, 'refunded');
564
+ INSERT INTO returns VALUES (2, 6, 'Wrong size', '2024-03-01', 129.99, 'refunded');
565
+ INSERT INTO returns VALUES (3, 10, 'Changed mind', '2024-03-25', 99.98, 'pending');
566
+ """
567
+
568
+ ECOMMERCE_DESCRIPTION = """DATABASE SCHEMA β€” E-Commerce Platform
569
+ =======================================
570
+
571
+ TABLE: sellers
572
+ - id (INTEGER, PK) | name (TEXT) | rating (REAL, 0.0-5.0) | city (TEXT)
573
+ - joined_date (TEXT)
574
+
575
+ TABLE: categories
576
+ - id (INTEGER, PK) | name (TEXT) | parent_category (TEXT, nullable β€” NULL = top-level)
577
+
578
+ TABLE: products
579
+ - id (INTEGER, PK) | name (TEXT) | category_id (FK→categories) | seller_id (FK→sellers)
580
+ - price (REAL) | stock (INTEGER) | listed_date (TEXT)
581
+
582
+ TABLE: users
583
+ - id (INTEGER, PK) | name (TEXT) | email (TEXT) | city (TEXT)
584
+ - signup_date (TEXT) | is_premium (INTEGER, 0/1)
585
+
586
+ TABLE: orders
587
+ - id (INTEGER, PK) | user_id (FK→users) | order_date (TEXT)
588
+ - total_amount (REAL) | status (TEXT, 'pending'/'shipped'/'delivered'/'cancelled')
589
+ - payment_method (TEXT, 'credit_card'/'debit_card'/'paypal')
590
+
591
+ TABLE: order_items
592
+ - id (INTEGER, PK) | order_id (FK→orders) | product_id (FK→products)
593
+ - quantity (INTEGER) | unit_price (REAL)
594
+
595
+ TABLE: returns
596
+ - id (INTEGER, PK) | order_item_id (FK→order_items) | reason (TEXT)
597
+ - return_date (TEXT) | refund_amount (REAL)
598
+ - status (TEXT, 'pending'/'refunded'/'rejected')
599
+
600
+ RELATIONSHIPS:
601
+ products.category_id β†’ categories.id
602
+ products.seller_id β†’ sellers.id
603
+ orders.user_id β†’ users.id
604
+ order_items.order_id β†’ orders.id
605
+ order_items.product_id β†’ products.id
606
+ returns.order_item_id β†’ order_items.id
607
+
608
+ NOTE: Categories have a parent_category for hierarchy (e.g., Smartphones β†’ Electronics).
609
+ Returns reference order_items, not orders directly.
610
+ """
611
+
612
+
613
+ # ---------------------------------------------------------------------------
614
+ # Schema registry
615
+ # ---------------------------------------------------------------------------
616
+
617
+ SCHEMAS: Dict[str, DatabaseSchema] = {
618
+ "company": DatabaseSchema(
619
+ id="company",
620
+ name="Company Analytics",
621
+ schema_sql=COMPANY_SCHEMA_SQL,
622
+ seed_sql=COMPANY_SEED_SQL,
623
+ description=COMPANY_DESCRIPTION,
624
+ ),
625
+ "hospital": DatabaseSchema(
626
+ id="hospital",
627
+ name="Hospital Management",
628
+ schema_sql=HOSPITAL_SCHEMA_SQL,
629
+ seed_sql=HOSPITAL_SEED_SQL,
630
+ description=HOSPITAL_DESCRIPTION,
631
+ ),
632
+ "ecommerce": DatabaseSchema(
633
+ id="ecommerce",
634
+ name="E-Commerce Platform",
635
+ schema_sql=ECOMMERCE_SCHEMA_SQL,
636
+ seed_sql=ECOMMERCE_SEED_SQL,
637
+ description=ECOMMERCE_DESCRIPTION,
638
+ ),
639
+ }
640
+
641
+
642
+ def create_database(schema_id: str = "company") -> sqlite3.Connection:
643
+ """Create an in-memory SQLite database for a specific domain."""
644
+ schema = SCHEMAS[schema_id]
645
+ conn = sqlite3.connect(":memory:", check_same_thread=False)
646
+ conn.execute("PRAGMA foreign_keys = ON")
647
+ conn.executescript(schema.schema_sql)
648
+ conn.executescript(schema.seed_sql)
649
+ return conn
650
+
651
+
652
+ def get_expected_result(conn: sqlite3.Connection, query: str) -> List[Tuple]:
653
+ """Execute a query and return the result set."""
654
+ cursor = conn.execute(query)
655
+ return cursor.fetchall()
656
+
657
+
658
+ def get_column_names(conn: sqlite3.Connection, query: str) -> List[str]:
659
+ """Get column names from a query result."""
660
+ cursor = conn.execute(query)
661
+ return [desc[0] for desc in cursor.description] if cursor.description else []
662
+
663
+
664
+ # ---------------------------------------------------------------------------
665
+ # Company tasks
666
+ # ---------------------------------------------------------------------------
667
+
668
+ COMPANY_TASKS = [
669
+ # ── EASY ──
670
+ Task(
671
+ id="company_easy_1",
672
+ difficulty="easy",
673
+ schema_id="company",
674
+ question="List all active employees in the Engineering department, showing their name and salary, sorted by salary from highest to lowest.",
675
+ ground_truth_query="""
676
+ SELECT e.name, e.salary
677
+ FROM employees e
678
+ JOIN departments d ON e.department_id = d.id
679
+ WHERE d.name = 'Engineering' AND e.is_active = 1
680
+ ORDER BY e.salary DESC
681
+ """,
682
+ expected_columns=["name", "salary"],
683
+ description="Simple filtered query with sorting",
684
+ hints=["Use employees and departments tables", "Filter by department name and active status"],
685
+ max_steps=5,
686
+ ),
687
+ Task(
688
+ id="company_easy_2",
689
+ difficulty="easy",
690
+ schema_id="company",
691
+ question="Show all products in the 'Electronics' category with a price greater than $50, sorted by price ascending.",
692
+ ground_truth_query="""
693
+ SELECT name, price
694
+ FROM products
695
+ WHERE category = 'Electronics' AND price > 50
696
+ ORDER BY price ASC
697
+ """,
698
+ expected_columns=["name", "price"],
699
+ description="Single table filter and sort",
700
+ hints=["Use the products table", "Filter by category and price"],
701
+ max_steps=5,
702
+ ),
703
+ # ── MEDIUM ──
704
+ Task(
705
+ id="company_medium_1",
706
+ difficulty="medium",
707
+ schema_id="company",
708
+ question="Find the total revenue (sum of total_price) per product category for completed orders. Show the category and total revenue, sorted by revenue descending.",
709
+ ground_truth_query="""
710
+ SELECT p.category, SUM(o.total_price) AS total_revenue
711
+ FROM orders o
712
+ JOIN products p ON o.product_id = p.id
713
+ WHERE o.status = 'completed'
714
+ GROUP BY p.category
715
+ ORDER BY total_revenue DESC
716
+ """,
717
+ expected_columns=["category", "total_revenue"],
718
+ description="Aggregation with JOIN and GROUP BY",
719
+ hints=["Join orders with products", "Filter for completed orders", "Group by category"],
720
+ max_steps=6,
721
+ ),
722
+ Task(
723
+ id="company_medium_2",
724
+ difficulty="medium",
725
+ schema_id="company",
726
+ question="Find the average salary per department, but only show departments where the average salary exceeds $100,000. Show department name and average salary, sorted by average salary descending.",
727
+ ground_truth_query="""
728
+ SELECT d.name, AVG(e.salary) AS avg_salary
729
+ FROM employees e
730
+ JOIN departments d ON e.department_id = d.id
731
+ WHERE e.is_active = 1
732
+ GROUP BY d.name
733
+ HAVING AVG(e.salary) > 100000
734
+ ORDER BY avg_salary DESC
735
+ """,
736
+ expected_columns=["name", "avg_salary"],
737
+ description="GROUP BY with HAVING clause",
738
+ hints=["Join employees with departments", "Use GROUP BY and HAVING"],
739
+ max_steps=6,
740
+ ),
741
+ # ── HARD ──
742
+ Task(
743
+ id="company_hard_1",
744
+ difficulty="hard",
745
+ schema_id="company",
746
+ question="Calculate each department's salary budget utilization: show the department name, total active employee salaries, department budget, and the utilization percentage (total salaries / budget * 100, rounded to 1 decimal). Sort by utilization percentage descending.",
747
+ ground_truth_query="""
748
+ SELECT
749
+ d.name,
750
+ SUM(e.salary) AS total_salaries,
751
+ d.budget,
752
+ ROUND(SUM(e.salary) * 100.0 / d.budget, 1) AS utilization_pct
753
+ FROM departments d
754
+ JOIN employees e ON d.id = e.department_id
755
+ WHERE e.is_active = 1
756
+ GROUP BY d.id, d.name, d.budget
757
+ ORDER BY utilization_pct DESC
758
+ """,
759
+ expected_columns=["name", "total_salaries", "budget", "utilization_pct"],
760
+ description="Computed columns with percentage calculations",
761
+ hints=[
762
+ "Join departments with employees",
763
+ "Filter active employees only",
764
+ "Compute: total_salaries / budget * 100",
765
+ ],
766
+ max_steps=8,
767
+ challenge_type="computation",
768
+ ),
769
+ ]
770
+
771
+ # ---------------------------------------------------------------------------
772
+ # Hospital tasks
773
+ # ---------------------------------------------------------------------------
774
+
775
+ HOSPITAL_TASKS = [
776
+ # ── EASY ──
777
+ Task(
778
+ id="hospital_easy_1",
779
+ difficulty="easy",
780
+ schema_id="hospital",
781
+ question="List all patients who are currently still admitted (discharge_date is NULL). Show their name, diagnosis, and admission date, sorted by admission date.",
782
+ ground_truth_query="""
783
+ SELECT name, diagnosis, admission_date
784
+ FROM patients
785
+ WHERE discharge_date IS NULL
786
+ ORDER BY admission_date ASC
787
+ """,
788
+ expected_columns=["name", "diagnosis", "admission_date"],
789
+ description="NULL handling filter",
790
+ hints=["Use the patients table", "NULL means still admitted β€” use IS NULL"],
791
+ max_steps=5,
792
+ challenge_type="null_handling",
793
+ ),
794
+ # ── MEDIUM ──
795
+ Task(
796
+ id="hospital_medium_1",
797
+ difficulty="medium",
798
+ schema_id="hospital",
799
+ question="Find the total medication cost for each currently admitted patient. Show the patient name, number of active prescriptions (end_date IS NULL), and the total daily medication cost (sum of medication unit_cost). Sort by total cost descending.",
800
+ ground_truth_query="""
801
+ SELECT
802
+ p.name,
803
+ COUNT(pr.id) AS active_prescriptions,
804
+ SUM(m.unit_cost) AS daily_med_cost
805
+ FROM patients p
806
+ JOIN prescriptions pr ON p.id = pr.patient_id
807
+ JOIN medications m ON pr.medication_id = m.id
808
+ WHERE p.discharge_date IS NULL AND pr.end_date IS NULL
809
+ GROUP BY p.id, p.name
810
+ ORDER BY daily_med_cost DESC
811
+ """,
812
+ expected_columns=["name", "active_prescriptions", "daily_med_cost"],
813
+ description="Multi-table JOIN with NULL filters and aggregation",
814
+ hints=[
815
+ "Join patients, prescriptions, and medications",
816
+ "Filter: patient still admitted (discharge_date IS NULL)",
817
+ "Filter: prescription still active (end_date IS NULL)",
818
+ ],
819
+ max_steps=7,
820
+ challenge_type="multi_null",
821
+ ),
822
+ # ── HARD ──
823
+ Task(
824
+ id="hospital_hard_1",
825
+ difficulty="hard",
826
+ schema_id="hospital",
827
+ question="Find doctors who have prescribed medications to patients outside their own ward. Show the doctor's name, their ward name, the patient's name, the patient's ward name, and the medication prescribed. Sort by doctor name.",
828
+ ground_truth_query="""
829
+ SELECT
830
+ d.name AS doctor_name,
831
+ wd.name AS doctor_ward,
832
+ p.name AS patient_name,
833
+ wp.name AS patient_ward,
834
+ m.name AS medication
835
+ FROM prescriptions pr
836
+ JOIN doctors d ON pr.doctor_id = d.id
837
+ JOIN patients p ON pr.patient_id = p.id
838
+ JOIN medications m ON pr.medication_id = m.id
839
+ JOIN wards wd ON d.ward_id = wd.id
840
+ JOIN wards wp ON p.ward_id = wp.id
841
+ WHERE d.ward_id != p.ward_id
842
+ ORDER BY d.name
843
+ """,
844
+ expected_columns=["doctor_name", "doctor_ward", "patient_name", "patient_ward", "medication"],
845
+ description="Cross-ward prescription analysis β€” 5-table JOIN with self-referencing comparison",
846
+ hints=[
847
+ "Join prescriptions with doctors, patients, medications, and wards (twice!)",
848
+ "Join wards once for doctor's ward and once for patient's ward",
849
+ "Compare doctor's ward_id with patient's ward_id",
850
+ ],
851
+ max_steps=10,
852
+ challenge_type="multi_join_complex",
853
+ ),
854
+ ]
855
+
856
+ # ---------------------------------------------------------------------------
857
+ # E-Commerce tasks
858
+ # ---------------------------------------------------------------------------
859
+
860
+ ECOMMERCE_TASKS = [
861
+ # ── EASY ──
862
+ Task(
863
+ id="ecommerce_easy_1",
864
+ difficulty="easy",
865
+ schema_id="ecommerce",
866
+ question="List all premium users and the city they are from, along with their signup date. Sort by signup date, most recent first.",
867
+ ground_truth_query="""
868
+ SELECT name, city, signup_date
869
+ FROM users
870
+ WHERE is_premium = 1
871
+ ORDER BY signup_date DESC
872
+ """,
873
+ expected_columns=["name", "city", "signup_date"],
874
+ description="Simple filter on single table",
875
+ hints=["Use the users table", "Filter by is_premium = 1"],
876
+ max_steps=5,
877
+ ),
878
+ # ── MEDIUM ──
879
+ Task(
880
+ id="ecommerce_medium_1",
881
+ difficulty="medium",
882
+ schema_id="ecommerce",
883
+ question="Find the total revenue per seller for delivered orders only. Show the seller name, seller rating, number of items sold, and total revenue. Sort by total revenue descending.",
884
+ ground_truth_query="""
885
+ SELECT
886
+ s.name,
887
+ s.rating,
888
+ SUM(oi.quantity) AS items_sold,
889
+ SUM(oi.quantity * oi.unit_price) AS total_revenue
890
+ FROM sellers s
891
+ JOIN products p ON s.id = p.seller_id
892
+ JOIN order_items oi ON p.id = oi.product_id
893
+ JOIN orders o ON oi.order_id = o.id
894
+ WHERE o.status = 'delivered'
895
+ GROUP BY s.id, s.name, s.rating
896
+ ORDER BY total_revenue DESC
897
+ """,
898
+ expected_columns=["name", "rating", "items_sold", "total_revenue"],
899
+ description="4-table JOIN with aggregation and status filter",
900
+ hints=[
901
+ "Chain: sellers β†’ products β†’ order_items β†’ orders",
902
+ "Filter for delivered orders",
903
+ "Group by seller",
904
+ ],
905
+ max_steps=7,
906
+ ),
907
+ # ── HARD ──
908
+ Task(
909
+ id="ecommerce_hard_1",
910
+ difficulty="hard",
911
+ schema_id="ecommerce",
912
+ question="Calculate the return rate for each product category. Show the category name, total items sold (from delivered orders), total items returned, and the return rate as a percentage (rounded to 1 decimal). Include categories with zero returns. Sort by return rate descending.",
913
+ ground_truth_query="""
914
+ SELECT
915
+ c.name AS category,
916
+ COALESCE(SUM(oi.quantity), 0) AS total_sold,
917
+ COALESCE(returned.total_returned, 0) AS total_returned,
918
+ ROUND(
919
+ COALESCE(returned.total_returned, 0) * 100.0 /
920
+ CASE WHEN SUM(oi.quantity) = 0 THEN 1 ELSE SUM(oi.quantity) END,
921
+ 1
922
+ ) AS return_rate_pct
923
+ FROM categories c
924
+ JOIN products p ON c.id = p.category_id
925
+ JOIN order_items oi ON p.id = oi.product_id
926
+ JOIN orders o ON oi.order_id = o.id
927
+ LEFT JOIN (
928
+ SELECT c2.id AS cat_id, COUNT(r.id) AS total_returned
929
+ FROM returns r
930
+ JOIN order_items oi2 ON r.order_item_id = oi2.id
931
+ JOIN products p2 ON oi2.product_id = p2.id
932
+ JOIN categories c2 ON p2.category_id = c2.id
933
+ GROUP BY c2.id
934
+ ) returned ON c.id = returned.cat_id
935
+ WHERE o.status = 'delivered'
936
+ GROUP BY c.id, c.name, returned.total_returned
937
+ ORDER BY return_rate_pct DESC
938
+ """,
939
+ expected_columns=["category", "total_sold", "total_returned", "return_rate_pct"],
940
+ description="Return rate analysis β€” subquery + LEFT JOIN + COALESCE + division-by-zero handling",
941
+ hints=[
942
+ "You need to count items sold per category AND items returned per category",
943
+ "Use a subquery or LEFT JOIN for returns (some categories have zero returns)",
944
+ "Handle division by zero with CASE WHEN",
945
+ "Use COALESCE for NULL return counts",
946
+ ],
947
+ max_steps=10,
948
+ challenge_type="subquery_with_edge_cases",
949
+ ),
950
+ ]
951
+
952
+
953
+ # ---------------------------------------------------------------------------
954
+ # Expert tasks β€” designed to genuinely challenge frontier models
955
+ # ---------------------------------------------------------------------------
956
+
957
+ EXPERT_TASKS = [
958
+ # Self-join: employees vs their managers (company)
959
+ Task(
960
+ id="company_hard_2",
961
+ difficulty="hard",
962
+ schema_id="company",
963
+ question="Find employees who earn more than their direct manager. Show the employee name, employee salary, manager name, and manager salary. Only include active employees with active managers. Sort by the salary difference (employee salary minus manager salary) descending.",
964
+ ground_truth_query="""
965
+ SELECT
966
+ e.name AS employee_name,
967
+ e.salary AS employee_salary,
968
+ m.name AS manager_name,
969
+ m.salary AS manager_salary
970
+ FROM employees e
971
+ JOIN employees m ON e.manager_id = m.id
972
+ WHERE e.is_active = 1 AND m.is_active = 1
973
+ AND e.salary > m.salary
974
+ ORDER BY (e.salary - m.salary) DESC
975
+ """,
976
+ expected_columns=["employee_name", "employee_salary", "manager_name", "manager_salary"],
977
+ description="Self-join with comparison across hierarchical relationship",
978
+ hints=[
979
+ "You need to join the employees table with itself",
980
+ "Alias one as 'e' (employee) and the other as 'm' (manager)",
981
+ "The manager_id column links employee to manager",
982
+ "Compare salaries between the employee row and the manager row",
983
+ ],
984
+ max_steps=8,
985
+ challenge_type="self_join",
986
+ ),
987
+
988
+ # Temporal reasoning: readmission within 30 days (hospital)
989
+ Task(
990
+ id="hospital_hard_2",
991
+ difficulty="hard",
992
+ schema_id="hospital",
993
+ question="Find patients who had multiple appointments with the same doctor. Show the patient name, doctor name, number of appointments together, and the date range (earliest to latest appointment date). Only count completed appointments. Sort by number of appointments descending.",
994
+ ground_truth_query="""
995
+ SELECT
996
+ p.name AS patient_name,
997
+ d.name AS doctor_name,
998
+ COUNT(a.id) AS visit_count,
999
+ MIN(a.appointment_date) AS first_visit,
1000
+ MAX(a.appointment_date) AS last_visit
1001
+ FROM appointments a
1002
+ JOIN patients p ON a.patient_id = p.id
1003
+ JOIN doctors d ON a.doctor_id = d.id
1004
+ WHERE a.status = 'completed'
1005
+ GROUP BY p.id, p.name, d.id, d.name
1006
+ HAVING COUNT(a.id) > 1
1007
+ ORDER BY visit_count DESC
1008
+ """,
1009
+ expected_columns=["patient_name", "doctor_name", "visit_count", "first_visit", "last_visit"],
1010
+ description="Temporal analysis with GROUP BY HAVING on joined data",
1011
+ hints=[
1012
+ "Join appointments with patients and doctors",
1013
+ "Group by patient-doctor pairs",
1014
+ "Use COUNT to find pairs with more than one appointment",
1015
+ "Use MIN/MAX for date range",
1016
+ ],
1017
+ max_steps=8,
1018
+ challenge_type="temporal_grouping",
1019
+ ),
1020
+
1021
+ # Multi-step: products in stock but never ordered (ecommerce)
1022
+ Task(
1023
+ id="ecommerce_hard_2",
1024
+ difficulty="hard",
1025
+ schema_id="ecommerce",
1026
+ question="Find products that are currently in stock (stock > 0) but have never been ordered. Show the product name, category name, seller name, price, and stock count. Sort by stock count descending. This helps identify inventory that isn't moving.",
1027
+ ground_truth_query="""
1028
+ SELECT
1029
+ p.name AS product_name,
1030
+ c.name AS category_name,
1031
+ s.name AS seller_name,
1032
+ p.price,
1033
+ p.stock
1034
+ FROM products p
1035
+ JOIN categories c ON p.category_id = c.id
1036
+ JOIN sellers s ON p.seller_id = s.id
1037
+ LEFT JOIN order_items oi ON p.id = oi.product_id
1038
+ WHERE oi.id IS NULL AND p.stock > 0
1039
+ ORDER BY p.stock DESC
1040
+ """,
1041
+ expected_columns=["product_name", "category_name", "seller_name", "price", "stock"],
1042
+ description="LEFT JOIN with IS NULL to find non-matching rows",
1043
+ hints=[
1044
+ "Use LEFT JOIN between products and order_items",
1045
+ "A product with no orders will have NULL in the order_items columns",
1046
+ "Filter with WHERE oi.id IS NULL to find products never ordered",
1047
+ "Also filter for stock > 0",
1048
+ ],
1049
+ max_steps=8,
1050
+ challenge_type="anti_join",
1051
+ ),
1052
+ ]
1053
+
1054
+
1055
+ # ---------------------------------------------------------------------------
1056
+ # All tasks combined
1057
+ # ---------------------------------------------------------------------------
1058
+
1059
+ ALL_TASKS_LIST = COMPANY_TASKS + HOSPITAL_TASKS + ECOMMERCE_TASKS + EXPERT_TASKS
1060
+
1061
+ ALL_TASKS = {
1062
+ "easy": [t for t in ALL_TASKS_LIST if t.difficulty == "easy"],
1063
+ "medium": [t for t in ALL_TASKS_LIST if t.difficulty == "medium"],
1064
+ "hard": [t for t in ALL_TASKS_LIST if t.difficulty == "hard"],
1065
+ }
1066
+
1067
+ TASK_MAP: Dict[str, Task] = {t.id: t for t in ALL_TASKS_LIST}
1068
+
uv.lock ADDED
The diff for this file is too large to render. See raw diff