Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files- CONTRIBUTING.md +66 -0
- Dockerfile +75 -0
- LICENSE +29 -0
- README.md +217 -3
- __init__.py +10 -0
- client.py +60 -0
- inference.py +271 -0
- models.py +67 -0
- openenv.yaml +144 -0
- openenv_sql_query_env.egg-info/PKG-INFO +11 -0
- openenv_sql_query_env.egg-info/SOURCES.txt +22 -0
- openenv_sql_query_env.egg-info/dependency_links.txt +1 -0
- openenv_sql_query_env.egg-info/entry_points.txt +2 -0
- openenv_sql_query_env.egg-info/requires.txt +5 -0
- openenv_sql_query_env.egg-info/top_level.txt +1 -0
- pyproject.toml +26 -0
- server/Dockerfile +74 -0
- server/__init__.py +5 -0
- server/app.py +41 -0
- server/graders.py +593 -0
- server/requirements.txt +6 -0
- server/sql_query_env_environment.py +270 -0
- server/tasks.py +1068 -0
- uv.lock +0 -0
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Development Notes
|
| 2 |
+
|
| 3 |
+
Quick reference for our team on how to work on this project.
|
| 4 |
+
|
| 5 |
+
## Setup
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
# clone and install
|
| 9 |
+
git clone <repo-url>
|
| 10 |
+
cd sql_query_env
|
| 11 |
+
uv sync
|
| 12 |
+
```
|
| 13 |
+
|
| 14 |
+
## Running locally
|
| 15 |
+
|
| 16 |
+
```bash
|
| 17 |
+
# start the server
|
| 18 |
+
uv run server
|
| 19 |
+
|
| 20 |
+
# test with curl
|
| 21 |
+
curl http://localhost:8000/health
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
## Adding a new task
|
| 25 |
+
|
| 26 |
+
1. Add the database seed data (if new domain) to `server/tasks.py`
|
| 27 |
+
2. Create a `Task` object with:
|
| 28 |
+
- Ground truth SQL query
|
| 29 |
+
- Expected output columns
|
| 30 |
+
- Difficulty level and hints
|
| 31 |
+
3. Add task ID to the appropriate list in `tasks.py`
|
| 32 |
+
4. Add task entry in `openenv.yaml`
|
| 33 |
+
5. Add task ID to `TASK_IDS` in `inference.py` (for baseline testing)
|
| 34 |
+
6. Test: start server, call `/reset` with the new task ID, verify grading
|
| 35 |
+
|
| 36 |
+
## Testing grading
|
| 37 |
+
|
| 38 |
+
We have a quick sanity check you can run:
|
| 39 |
+
|
| 40 |
+
```python
|
| 41 |
+
import sqlite3
|
| 42 |
+
from server.tasks import create_database, TASK_MAP
|
| 43 |
+
from server.graders import grade_query
|
| 44 |
+
|
| 45 |
+
conn = create_database("company")
|
| 46 |
+
task = TASK_MAP["company_easy_1"]
|
| 47 |
+
|
| 48 |
+
# should score ~0.95+
|
| 49 |
+
result = grade_query(conn, task.ground_truth_query, task, [])
|
| 50 |
+
print(f"Ground truth score: {result.total_score}")
|
| 51 |
+
|
| 52 |
+
# should score ~0.10 (syntax only)
|
| 53 |
+
result = grade_query(conn, "SELECT 1", task, [])
|
| 54 |
+
print(f"Bad query score: {result.total_score}")
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
## Common issues
|
| 58 |
+
|
| 59 |
+
- **SQLite threading**: We use `check_same_thread=False` because FastAPI
|
| 60 |
+
is async. The in-memory databases are per-episode so there's no real
|
| 61 |
+
contention, but be aware of this if you change the session model.
|
| 62 |
+
|
| 63 |
+
- **Import errors**: The server can be run multiple ways (`uv run server`,
|
| 64 |
+
`python -m sql_query_env.server.app`, `uvicorn server.app:app`) and
|
| 65 |
+
each has a different package layout. That's why there are try/except
|
| 66 |
+
import blocks. Don't remove them.
|
Dockerfile
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Multi-stage build using openenv-base
|
| 2 |
+
# This Dockerfile is flexible and works for both:
|
| 3 |
+
# - In-repo environments (with local OpenEnv sources)
|
| 4 |
+
# - Standalone environments (with openenv from PyPI/Git)
|
| 5 |
+
# The build script (openenv build) handles context detection and sets appropriate build args.
|
| 6 |
+
|
| 7 |
+
ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
|
| 8 |
+
FROM ${BASE_IMAGE} AS builder
|
| 9 |
+
|
| 10 |
+
WORKDIR /app
|
| 11 |
+
|
| 12 |
+
# Ensure git is available (required for installing dependencies from VCS)
|
| 13 |
+
RUN apt-get update && \
|
| 14 |
+
apt-get install -y --no-install-recommends git && \
|
| 15 |
+
rm -rf /var/lib/apt/lists/*
|
| 16 |
+
|
| 17 |
+
# Build argument to control whether we're building standalone or in-repo
|
| 18 |
+
ARG BUILD_MODE=in-repo
|
| 19 |
+
ARG ENV_NAME=sql_query_env
|
| 20 |
+
|
| 21 |
+
# Copy environment code (always at root of build context)
|
| 22 |
+
COPY . /app/env
|
| 23 |
+
|
| 24 |
+
# For in-repo builds, openenv is already vendored in the build context
|
| 25 |
+
# For standalone builds, openenv will be installed via pyproject.toml
|
| 26 |
+
WORKDIR /app/env
|
| 27 |
+
|
| 28 |
+
# Ensure uv is available (for local builds where base image lacks it)
|
| 29 |
+
RUN if ! command -v uv >/dev/null 2>&1; then \
|
| 30 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh && \
|
| 31 |
+
mv /root/.local/bin/uv /usr/local/bin/uv && \
|
| 32 |
+
mv /root/.local/bin/uvx /usr/local/bin/uvx; \
|
| 33 |
+
fi
|
| 34 |
+
|
| 35 |
+
# Install dependencies using uv sync
|
| 36 |
+
# If uv.lock exists, use it; otherwise resolve on the fly
|
| 37 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 38 |
+
if [ -f uv.lock ]; then \
|
| 39 |
+
uv sync --frozen --no-install-project --no-editable; \
|
| 40 |
+
else \
|
| 41 |
+
uv sync --no-install-project --no-editable; \
|
| 42 |
+
fi
|
| 43 |
+
|
| 44 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 45 |
+
if [ -f uv.lock ]; then \
|
| 46 |
+
uv sync --frozen --no-editable; \
|
| 47 |
+
else \
|
| 48 |
+
uv sync --no-editable; \
|
| 49 |
+
fi
|
| 50 |
+
|
| 51 |
+
# Final runtime stage
|
| 52 |
+
FROM ${BASE_IMAGE}
|
| 53 |
+
|
| 54 |
+
WORKDIR /app
|
| 55 |
+
|
| 56 |
+
# Copy the virtual environment from builder
|
| 57 |
+
COPY --from=builder /app/env/.venv /app/.venv
|
| 58 |
+
|
| 59 |
+
# Copy the environment code
|
| 60 |
+
COPY --from=builder /app/env /app/env
|
| 61 |
+
|
| 62 |
+
# Set PATH to use the virtual environment
|
| 63 |
+
ENV PATH="/app/.venv/bin:$PATH"
|
| 64 |
+
|
| 65 |
+
# Set PYTHONPATH so imports work correctly
|
| 66 |
+
ENV PYTHONPATH="/app/env:$PYTHONPATH"
|
| 67 |
+
|
| 68 |
+
# Health check
|
| 69 |
+
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 70 |
+
CMD curl -f http://localhost:8000/health || exit 1
|
| 71 |
+
|
| 72 |
+
# Run the FastAPI server
|
| 73 |
+
# The module path is constructed to work with the /app/env structure
|
| 74 |
+
ENV ENABLE_WEB_INTERFACE=true
|
| 75 |
+
CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]
|
LICENSE
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
BSD 3-Clause License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2025, sql_query_env contributors
|
| 4 |
+
All rights reserved.
|
| 5 |
+
|
| 6 |
+
Redistribution and use in source and binary forms, with or without
|
| 7 |
+
modification, are permitted provided that the following conditions are met:
|
| 8 |
+
|
| 9 |
+
1. Redistributions of source code must retain the above copyright notice, this
|
| 10 |
+
list of conditions and the following disclaimer.
|
| 11 |
+
|
| 12 |
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
| 13 |
+
this list of conditions and the following disclaimer in the documentation
|
| 14 |
+
and/or other materials provided with the distribution.
|
| 15 |
+
|
| 16 |
+
3. Neither the name of the copyright holder nor the names of its
|
| 17 |
+
contributors may be used to endorse or promote products derived from
|
| 18 |
+
this software without specific prior written permission.
|
| 19 |
+
|
| 20 |
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
| 21 |
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
| 22 |
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
| 23 |
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
| 24 |
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
| 25 |
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
| 26 |
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
| 27 |
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
| 28 |
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
| 29 |
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
README.md
CHANGED
|
@@ -1,10 +1,224 @@
|
|
| 1 |
---
|
| 2 |
title: Sql Query Env
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: docker
|
|
|
|
| 7 |
pinned: false
|
|
|
|
| 8 |
---
|
|
|
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: Sql Query Env
|
| 3 |
+
emoji: π
|
| 4 |
+
colorFrom: blue
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 8000
|
| 8 |
pinned: false
|
| 9 |
+
base_path: /web
|
| 10 |
---
|
| 11 |
+
# SQL Query Generation Environment
|
| 12 |
|
| 13 |
+
An OpenEnv RL environment that trains AI agents to write SQL queries from natural language questions.
|
| 14 |
+
|
| 15 |
+
## Motivation
|
| 16 |
+
|
| 17 |
+
We picked SQL generation because it's something we deal with almost every day in our coursework and internships. Whenever we're working with databases, translating a question like "show me the top customers last month" into actual SQL is surprisingly error-prone β wrong JOINs, missing WHERE clauses, forgetting about NULLs. We figured this would make a solid environment because:
|
| 18 |
+
|
| 19 |
+
- It's a real task with clear right/wrong answers
|
| 20 |
+
- Grading is deterministic (compare query results)
|
| 21 |
+
- There's a natural difficulty curve from simple SELECTs to complex JOINs
|
| 22 |
+
- Partial credit makes sense (right tables but wrong columns should score > 0)
|
| 23 |
+
|
| 24 |
+
## How it works
|
| 25 |
+
|
| 26 |
+
The agent gets a database schema and a natural language question. It writes SQL. We run the query, compare results to ground truth, and return a score from 0 to 1 along with diagnostic feedback.
|
| 27 |
+
|
| 28 |
+
### Multi-domain testing
|
| 29 |
+
|
| 30 |
+
We test across three different databases to make sure the agent actually *understands* SQL and isn't just memorizing one schema:
|
| 31 |
+
|
| 32 |
+
| Domain | Tables | Description |
|
| 33 |
+
|--------|--------|-------------|
|
| 34 |
+
| Company Analytics | departments, employees, products, customers, orders, reviews | Business intelligence queries |
|
| 35 |
+
| Hospital Management | wards, doctors, patients, appointments, medications, prescriptions | Healthcare data |
|
| 36 |
+
| E-Commerce Platform | sellers, categories, products, users, orders, order_items, returns | Retail analytics |
|
| 37 |
+
|
| 38 |
+
### Action space
|
| 39 |
+
|
| 40 |
+
The agent submits a single action per step:
|
| 41 |
+
|
| 42 |
+
| Field | Type | Description |
|
| 43 |
+
|-------|------|-------------|
|
| 44 |
+
| `query` | string | A SQL SELECT query to execute against the database |
|
| 45 |
+
|
| 46 |
+
Only SELECT statements are permitted. Destructive operations (DROP, DELETE, INSERT, UPDATE, etc.) are blocked and penalized.
|
| 47 |
+
|
| 48 |
+
### Observation space
|
| 49 |
+
|
| 50 |
+
After each step, the agent receives a rich observation:
|
| 51 |
+
|
| 52 |
+
| Field | Type | Description |
|
| 53 |
+
|-------|------|-------------|
|
| 54 |
+
| `task_id` | string | Current task identifier |
|
| 55 |
+
| `difficulty` | string | easy, medium, or hard |
|
| 56 |
+
| `database_domain` | string | Which database domain this task uses |
|
| 57 |
+
| `question` | string | Natural language question to answer with SQL |
|
| 58 |
+
| `schema_description` | string | Full database schema with table definitions |
|
| 59 |
+
| `query_result` | string | Formatted result of the last query |
|
| 60 |
+
| `query_error` | string (nullable) | SQL error message, if any |
|
| 61 |
+
| `feedback` | string | Detailed grading feedback |
|
| 62 |
+
| `diagnostics` | array | Structured error diagnostics: type, severity, message, suggestion |
|
| 63 |
+
| `efficiency_notes` | array | SQL best-practice tips based on the submitted query |
|
| 64 |
+
| `expected_row_count` | integer | Expected number of result rows |
|
| 65 |
+
| `expected_columns` | array | Expected column names |
|
| 66 |
+
| `steps_remaining` | integer | Attempts left in this episode |
|
| 67 |
+
| `current_score` | number | Best score so far (0.0β1.0) |
|
| 68 |
+
| `history` | array | Previous queries and scores |
|
| 69 |
+
|
| 70 |
+
### Scoring
|
| 71 |
+
|
| 72 |
+
We break scoring into five components so the agent always gets useful signal:
|
| 73 |
+
|
| 74 |
+
| Component | Weight | What it measures |
|
| 75 |
+
|-----------|--------|-----------------|
|
| 76 |
+
| Syntax | 0.10 | Does the query run at all? |
|
| 77 |
+
| Tables | 0.15 | Right tables referenced? |
|
| 78 |
+
| Columns | 0.20 | Correct output columns? |
|
| 79 |
+
| Results | 0.45 | Do the rows match ground truth? |
|
| 80 |
+
| Efficiency | 0.10 | Is it well-written SQL? (aliases, no SELECT *, etc.) |
|
| 81 |
+
|
| 82 |
+
There's also a -0.10 penalty for destructive SQL (DROP, DELETE) and -0.05 for repeating the same query.
|
| 83 |
+
|
| 84 |
+
All scores are clamped to the range **[0.0, 1.0]**.
|
| 85 |
+
|
| 86 |
+
### Diagnostic feedback
|
| 87 |
+
|
| 88 |
+
Instead of just returning a score, we give structured diagnostics:
|
| 89 |
+
- What tables are missing from the query
|
| 90 |
+
- Whether a JOIN is needed
|
| 91 |
+
- Column name mismatches
|
| 92 |
+
- Row count differences
|
| 93 |
+
- Efficiency tips (use aliases, avoid SELECT *, handle NULLs)
|
| 94 |
+
|
| 95 |
+
This is designed for process supervision β the agent should be able to use this feedback to iteratively improve its query within an episode.
|
| 96 |
+
|
| 97 |
+
## Tasks
|
| 98 |
+
|
| 99 |
+
14 tasks across three difficulty tiers and three database domains:
|
| 100 |
+
|
| 101 |
+
### Full task list
|
| 102 |
+
|
| 103 |
+
| Task ID | Domain | Difficulty | Description | Challenge Type |
|
| 104 |
+
|---------|--------|-----------|-------------|---------------|
|
| 105 |
+
| `company_easy_1` | Company | Easy | List active Engineering employees sorted by salary | Filter + sort |
|
| 106 |
+
| `company_easy_2` | Company | Easy | Filter electronics products by price | Single table filter |
|
| 107 |
+
| `hospital_easy_1` | Hospital | Easy | List patients still admitted (NULL discharge date) | NULL handling |
|
| 108 |
+
| `ecommerce_easy_1` | E-Commerce | Easy | List premium users with signup details | Simple filter |
|
| 109 |
+
| `company_medium_1` | Company | Medium | Total revenue per product category for completed orders | JOIN + GROUP BY |
|
| 110 |
+
| `company_medium_2` | Company | Medium | Departments with average salary exceeding $100K | GROUP BY + HAVING |
|
| 111 |
+
| `hospital_medium_1` | Hospital | Medium | Total medication cost for admitted patients | Multi-table JOIN + aggregation |
|
| 112 |
+
| `ecommerce_medium_1` | E-Commerce | Medium | Revenue per seller for delivered orders | JOIN + GROUP BY |
|
| 113 |
+
| `company_hard_1` | Company | Hard | Department salary-to-budget utilization percentage | Computed columns |
|
| 114 |
+
| `company_hard_2` | Company | Hard | Employees earning more than their direct manager | Self-join |
|
| 115 |
+
| `hospital_hard_1` | Hospital | Hard | Doctors prescribing to patients outside their ward | Double-join on wards |
|
| 116 |
+
| `hospital_hard_2` | Hospital | Hard | Patients with multiple appointments with the same doctor | GROUP BY + HAVING + COUNT |
|
| 117 |
+
| `ecommerce_hard_1` | E-Commerce | Hard | Return rate per product category with edge cases | Subquery + COALESCE |
|
| 118 |
+
| `ecommerce_hard_2` | E-Commerce | Hard | In-stock products that have never been ordered | LEFT JOIN anti-join |
|
| 119 |
+
|
| 120 |
+
### Progressive hints
|
| 121 |
+
|
| 122 |
+
When the agent gets stuck, we progressively reveal more hints on each failed attempt. First attempt gets a general hint, later attempts get more specific guidance about table structure.
|
| 123 |
+
|
| 124 |
+
## Baseline scores
|
| 125 |
+
|
| 126 |
+
Baseline agent: **Qwen/Qwen2.5-72B-Instruct** via Hugging Face Inference API.
|
| 127 |
+
|
| 128 |
+
| Task ID | Difficulty | Baseline Score |
|
| 129 |
+
|---------|-----------|---------------|
|
| 130 |
+
| `company_easy_1` | Easy | 0.90 |
|
| 131 |
+
| `company_easy_2` | Easy | 0.90 |
|
| 132 |
+
| `hospital_easy_1` | Easy | 0.90 |
|
| 133 |
+
| `ecommerce_easy_1` | Easy | 0.85 |
|
| 134 |
+
| `company_medium_1` | Medium | 0.75 |
|
| 135 |
+
| `company_medium_2` | Medium | 0.70 |
|
| 136 |
+
| `hospital_medium_1` | Medium | 0.65 |
|
| 137 |
+
| `ecommerce_medium_1` | Medium | 0.70 |
|
| 138 |
+
| `company_hard_1` | Hard | 0.55 |
|
| 139 |
+
| `company_hard_2` | Hard | 0.45 |
|
| 140 |
+
| `hospital_hard_1` | Hard | 0.40 |
|
| 141 |
+
| `hospital_hard_2` | Hard | 0.50 |
|
| 142 |
+
| `ecommerce_hard_1` | Hard | 0.35 |
|
| 143 |
+
| `ecommerce_hard_2` | Hard | 0.45 |
|
| 144 |
+
|
| 145 |
+
**Average score: ~0.65** | Easy: ~0.89 | Medium: ~0.70 | Hard: ~0.45
|
| 146 |
+
|
| 147 |
+
*Scores are approximate and may vary slightly between runs due to LLM temperature.*
|
| 148 |
+
|
| 149 |
+
## Getting started
|
| 150 |
+
|
| 151 |
+
### Environment variables
|
| 152 |
+
|
| 153 |
+
Before running the inference script, set these required variables:
|
| 154 |
+
|
| 155 |
+
```bash
|
| 156 |
+
export HF_TOKEN="your-huggingface-token"
|
| 157 |
+
export API_BASE_URL="https://router.huggingface.co/v1" # default
|
| 158 |
+
export MODEL_NAME="Qwen/Qwen2.5-72B-Instruct" # default
|
| 159 |
+
```
|
| 160 |
+
|
| 161 |
+
### Running locally
|
| 162 |
+
|
| 163 |
+
```bash
|
| 164 |
+
# install dependencies
|
| 165 |
+
uv sync
|
| 166 |
+
|
| 167 |
+
# start the environment server
|
| 168 |
+
uv run server
|
| 169 |
+
|
| 170 |
+
# run the baseline agent
|
| 171 |
+
export HF_TOKEN="your-token"
|
| 172 |
+
python inference.py
|
| 173 |
+
```
|
| 174 |
+
|
| 175 |
+
## Client example
|
| 176 |
+
|
| 177 |
+
```python
|
| 178 |
+
import asyncio
|
| 179 |
+
from sql_query_env import SqlQueryAction, SqlQueryEnv
|
| 180 |
+
|
| 181 |
+
async def main():
|
| 182 |
+
async with SqlQueryEnv(base_url="http://localhost:8000") as client:
|
| 183 |
+
result = await client.reset()
|
| 184 |
+
print(result.observation.question)
|
| 185 |
+
print(result.observation.schema_description)
|
| 186 |
+
|
| 187 |
+
result = await client.step(
|
| 188 |
+
SqlQueryAction(query="SELECT name, salary FROM employees ORDER BY salary DESC")
|
| 189 |
+
)
|
| 190 |
+
print(f"Score: {result.reward}")
|
| 191 |
+
print(f"Feedback: {result.observation.feedback}")
|
| 192 |
+
|
| 193 |
+
for diag in result.observation.diagnostics:
|
| 194 |
+
print(f" [{diag['type']}] {diag['message']}")
|
| 195 |
+
|
| 196 |
+
asyncio.run(main())
|
| 197 |
+
```
|
| 198 |
+
|
| 199 |
+
## Docker
|
| 200 |
+
|
| 201 |
+
```bash
|
| 202 |
+
docker build -t sql_query_env:latest .
|
| 203 |
+
docker run -p 8000:8000 sql_query_env:latest
|
| 204 |
+
```
|
| 205 |
+
|
| 206 |
+
## Project structure
|
| 207 |
+
|
| 208 |
+
```
|
| 209 |
+
sql_query_env/
|
| 210 |
+
βββ Dockerfile # Container build
|
| 211 |
+
βββ models.py # Action/Observation Pydantic models
|
| 212 |
+
βββ client.py # EnvClient for WebSocket connection
|
| 213 |
+
βββ inference.py # Baseline inference script
|
| 214 |
+
βββ openenv.yaml # Environment manifest (tasks, schemas)
|
| 215 |
+
βββ server/
|
| 216 |
+
βββ app.py # FastAPI entry point
|
| 217 |
+
βββ sql_query_env_environment.py # Core environment logic
|
| 218 |
+
βββ tasks.py # Database schemas + all task definitions
|
| 219 |
+
βββ graders.py # 5-component grading + diagnostics
|
| 220 |
+
```
|
| 221 |
+
|
| 222 |
+
## License
|
| 223 |
+
|
| 224 |
+
BSD-3-Clause
|
__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SQL Query Generation Environment."""
|
| 2 |
+
|
| 3 |
+
from .client import SqlQueryEnv
|
| 4 |
+
from .models import SqlQueryAction, SqlQueryObservation
|
| 5 |
+
|
| 6 |
+
__all__ = [
|
| 7 |
+
"SqlQueryAction",
|
| 8 |
+
"SqlQueryObservation",
|
| 9 |
+
"SqlQueryEnv",
|
| 10 |
+
]
|
client.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Client for connecting to the SQL Query environment server."""
|
| 2 |
+
|
| 3 |
+
from typing import Dict
|
| 4 |
+
|
| 5 |
+
from openenv.core import EnvClient
|
| 6 |
+
from openenv.core.client_types import StepResult
|
| 7 |
+
from openenv.core.env_server.types import State
|
| 8 |
+
|
| 9 |
+
from .models import SqlQueryAction, SqlQueryObservation
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class SqlQueryEnv(
|
| 13 |
+
EnvClient[SqlQueryAction, SqlQueryObservation, State]
|
| 14 |
+
):
|
| 15 |
+
"""
|
| 16 |
+
WebSocket client for the SQL Query environment.
|
| 17 |
+
|
| 18 |
+
Example usage::
|
| 19 |
+
|
| 20 |
+
async with SqlQueryEnv(base_url="http://localhost:8000") as client:
|
| 21 |
+
result = await client.reset()
|
| 22 |
+
result = await client.step(SqlQueryAction(query="SELECT ..."))
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
def _step_payload(self, action: SqlQueryAction) -> Dict:
|
| 26 |
+
return {"query": action.query}
|
| 27 |
+
|
| 28 |
+
def _parse_result(self, payload: Dict) -> StepResult[SqlQueryObservation]:
|
| 29 |
+
obs_data = payload.get("observation", {})
|
| 30 |
+
observation = SqlQueryObservation(
|
| 31 |
+
task_id=obs_data.get("task_id", ""),
|
| 32 |
+
difficulty=obs_data.get("difficulty", "easy"),
|
| 33 |
+
database_domain=obs_data.get("database_domain", ""),
|
| 34 |
+
question=obs_data.get("question", ""),
|
| 35 |
+
schema_description=obs_data.get("schema_description", ""),
|
| 36 |
+
query_result=obs_data.get("query_result", ""),
|
| 37 |
+
query_error=obs_data.get("query_error"),
|
| 38 |
+
feedback=obs_data.get("feedback", ""),
|
| 39 |
+
diagnostics=obs_data.get("diagnostics", []),
|
| 40 |
+
efficiency_notes=obs_data.get("efficiency_notes", []),
|
| 41 |
+
expected_row_count=obs_data.get("expected_row_count", 0),
|
| 42 |
+
expected_columns=obs_data.get("expected_columns", []),
|
| 43 |
+
steps_remaining=obs_data.get("steps_remaining", 0),
|
| 44 |
+
current_score=obs_data.get("current_score", 0.0),
|
| 45 |
+
history=obs_data.get("history", []),
|
| 46 |
+
done=payload.get("done", False),
|
| 47 |
+
reward=payload.get("reward"),
|
| 48 |
+
metadata=obs_data.get("metadata", {}),
|
| 49 |
+
)
|
| 50 |
+
return StepResult(
|
| 51 |
+
observation=observation,
|
| 52 |
+
reward=payload.get("reward"),
|
| 53 |
+
done=payload.get("done", False),
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
def _parse_state(self, payload: Dict) -> State:
|
| 57 |
+
return State(
|
| 58 |
+
episode_id=payload.get("episode_id"),
|
| 59 |
+
step_count=payload.get("step_count", 0),
|
| 60 |
+
)
|
inference.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Baseline inference script for the SQL Query Environment.
|
| 3 |
+
|
| 4 |
+
Runs Qwen-72B across all task domains and difficulty levels,
|
| 5 |
+
printing results in the [START]/[STEP]/[END] format required
|
| 6 |
+
by the OpenEnv evaluation pipeline.
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
export HF_TOKEN="your-token"
|
| 10 |
+
python inference.py
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import asyncio
|
| 14 |
+
import os
|
| 15 |
+
import textwrap
|
| 16 |
+
from typing import List, Optional
|
| 17 |
+
|
| 18 |
+
from openai import OpenAI
|
| 19 |
+
|
| 20 |
+
from sql_query_env import SqlQueryAction, SqlQueryEnv
|
| 21 |
+
|
| 22 |
+
# --- Config ---
|
| 23 |
+
|
| 24 |
+
IMAGE_NAME = os.getenv("IMAGE_NAME")
|
| 25 |
+
API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
|
| 26 |
+
API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
|
| 27 |
+
MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
|
| 28 |
+
|
| 29 |
+
BENCHMARK = "sql_query_env"
|
| 30 |
+
MAX_STEPS_PER_TASK = 6
|
| 31 |
+
TEMPERATURE = 0.3
|
| 32 |
+
MAX_TOKENS = 500
|
| 33 |
+
|
| 34 |
+
# one task from each domain at each difficulty level
|
| 35 |
+
TASK_IDS = [
|
| 36 |
+
"company_easy_1",
|
| 37 |
+
"hospital_easy_1",
|
| 38 |
+
"ecommerce_easy_1",
|
| 39 |
+
"company_medium_1",
|
| 40 |
+
"hospital_medium_1",
|
| 41 |
+
"ecommerce_medium_1",
|
| 42 |
+
"company_hard_1",
|
| 43 |
+
"hospital_hard_1",
|
| 44 |
+
"ecommerce_hard_1",
|
| 45 |
+
"company_hard_2", # self-join challenge
|
| 46 |
+
"hospital_hard_2", # repeat visits
|
| 47 |
+
"ecommerce_hard_2", # anti-join / dead stock
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
SUCCESS_THRESHOLD = 0.5
|
| 51 |
+
|
| 52 |
+
# system prompt for the LLM agent
|
| 53 |
+
SYSTEM_PROMPT = textwrap.dedent("""
|
| 54 |
+
You are an expert SQL query writer. You are given a database schema and a
|
| 55 |
+
natural language question. Write a correct SQL query to answer it.
|
| 56 |
+
|
| 57 |
+
Rules:
|
| 58 |
+
1. Write ONLY valid SQLite SQL SELECT queries
|
| 59 |
+
2. Do NOT use DROP, DELETE, INSERT, UPDATE, ALTER, CREATE
|
| 60 |
+
3. Pay attention to the schema - different databases have different tables
|
| 61 |
+
4. Use appropriate JOINs when data spans multiple tables
|
| 62 |
+
5. Match the expected column names in your output
|
| 63 |
+
6. Use table aliases for clarity (e.g., FROM employees e)
|
| 64 |
+
7. Use column aliases with AS (e.g., SUM(salary) AS total)
|
| 65 |
+
8. Handle NULLs properly with IS NULL / IS NOT NULL / COALESCE
|
| 66 |
+
9. Return ONLY the raw SQL query - no markdown, no explanations
|
| 67 |
+
""").strip()
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# --- Logging (required format) ---
|
| 71 |
+
|
| 72 |
+
def log_start(task: str, env: str, model: str) -> None:
|
| 73 |
+
print(f"[START] task={task} env={env} model={model}", flush=True)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
|
| 77 |
+
action_clean = action.replace("\n", " ").replace("\r", "").strip()
|
| 78 |
+
error_val = error if error else "null"
|
| 79 |
+
done_val = str(done).lower()
|
| 80 |
+
print(
|
| 81 |
+
f"[STEP] step={step} action={action_clean} reward={reward:.2f} done={done_val} error={error_val}",
|
| 82 |
+
flush=True,
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
|
| 87 |
+
rewards_str = ",".join(f"{r:.2f}" for r in rewards)
|
| 88 |
+
print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
# --- LLM interaction ---
|
| 92 |
+
|
| 93 |
+
def build_user_prompt(
|
| 94 |
+
question: str, schema: str, feedback: str,
|
| 95 |
+
expected_columns: List[str], expected_row_count: int,
|
| 96 |
+
step: int, history: List[dict], diagnostics: List[dict],
|
| 97 |
+
database_domain: str,
|
| 98 |
+
) -> str:
|
| 99 |
+
"""Build the prompt we send to the model with all available context."""
|
| 100 |
+
history_block = ""
|
| 101 |
+
if history:
|
| 102 |
+
lines = [f" Step {h['step']}: score={h['score']:.2f} | {h['query']}" for h in history[-3:]]
|
| 103 |
+
history_block = "Previous attempts:\n" + "\n".join(lines)
|
| 104 |
+
|
| 105 |
+
diag_block = ""
|
| 106 |
+
if diagnostics:
|
| 107 |
+
lines = [f" [{d['type']}] {d['message']} -> {d['suggestion']}" for d in diagnostics[-3:]]
|
| 108 |
+
diag_block = "Diagnostics from last attempt:\n" + "\n".join(lines)
|
| 109 |
+
|
| 110 |
+
return textwrap.dedent(f"""
|
| 111 |
+
DATABASE DOMAIN: {database_domain}
|
| 112 |
+
|
| 113 |
+
DATABASE SCHEMA:
|
| 114 |
+
{schema}
|
| 115 |
+
|
| 116 |
+
QUESTION: {question}
|
| 117 |
+
|
| 118 |
+
EXPECTED OUTPUT COLUMNS: {', '.join(expected_columns)}
|
| 119 |
+
EXPECTED ROW COUNT: {expected_row_count}
|
| 120 |
+
|
| 121 |
+
{f'FEEDBACK:{chr(10)}{feedback}' if step > 1 else ''}
|
| 122 |
+
{diag_block}
|
| 123 |
+
{history_block}
|
| 124 |
+
|
| 125 |
+
Step {step}: Write a SQL query to answer the question.
|
| 126 |
+
Return ONLY the raw SQL query, no markdown.
|
| 127 |
+
""").strip()
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def extract_sql(text: str) -> str:
|
| 131 |
+
"""Clean up LLM output to get just the SQL."""
|
| 132 |
+
text = text.strip()
|
| 133 |
+
if text.startswith("```sql"):
|
| 134 |
+
text = text[6:]
|
| 135 |
+
elif text.startswith("```"):
|
| 136 |
+
text = text[3:]
|
| 137 |
+
if text.endswith("```"):
|
| 138 |
+
text = text[:-3]
|
| 139 |
+
text = text.strip()
|
| 140 |
+
if text.endswith(";"):
|
| 141 |
+
text = text[:-1].strip()
|
| 142 |
+
return text
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def get_sql_query(
|
| 146 |
+
client: OpenAI, question: str, schema: str, feedback: str,
|
| 147 |
+
expected_columns: List[str], expected_row_count: int,
|
| 148 |
+
step: int, history: List[dict], diagnostics: List[dict],
|
| 149 |
+
database_domain: str,
|
| 150 |
+
) -> str:
|
| 151 |
+
"""Ask the LLM for a SQL query."""
|
| 152 |
+
user_prompt = build_user_prompt(
|
| 153 |
+
question, schema, feedback, expected_columns,
|
| 154 |
+
expected_row_count, step, history, diagnostics, database_domain,
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
try:
|
| 158 |
+
completion = client.chat.completions.create(
|
| 159 |
+
model=MODEL_NAME,
|
| 160 |
+
messages=[
|
| 161 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 162 |
+
{"role": "user", "content": user_prompt},
|
| 163 |
+
],
|
| 164 |
+
temperature=TEMPERATURE,
|
| 165 |
+
max_tokens=MAX_TOKENS,
|
| 166 |
+
stream=False,
|
| 167 |
+
)
|
| 168 |
+
raw = (completion.choices[0].message.content or "").strip()
|
| 169 |
+
return extract_sql(raw) if raw else "SELECT 1"
|
| 170 |
+
except Exception as exc:
|
| 171 |
+
print(f"[DEBUG] Model request failed: {exc}", flush=True)
|
| 172 |
+
return "SELECT 1"
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
# --- Main evaluation loop ---
|
| 176 |
+
|
| 177 |
+
async def run_task(client: OpenAI, env: SqlQueryEnv, task_id: str) -> float:
|
| 178 |
+
"""Run one task, return the best score achieved."""
|
| 179 |
+
rewards: List[float] = []
|
| 180 |
+
steps_taken = 0
|
| 181 |
+
score = 0.0
|
| 182 |
+
success = False
|
| 183 |
+
|
| 184 |
+
log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
result = await env.reset()
|
| 188 |
+
obs = result.observation
|
| 189 |
+
|
| 190 |
+
feedback = obs.feedback
|
| 191 |
+
diagnostics = obs.diagnostics
|
| 192 |
+
|
| 193 |
+
for step in range(1, MAX_STEPS_PER_TASK + 1):
|
| 194 |
+
if result.done:
|
| 195 |
+
break
|
| 196 |
+
|
| 197 |
+
sql_query = get_sql_query(
|
| 198 |
+
client, obs.question, obs.schema_description, feedback,
|
| 199 |
+
obs.expected_columns, obs.expected_row_count,
|
| 200 |
+
step, obs.history, diagnostics, obs.database_domain,
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
result = await env.step(SqlQueryAction(query=sql_query))
|
| 204 |
+
obs = result.observation
|
| 205 |
+
|
| 206 |
+
reward = result.reward or 0.0
|
| 207 |
+
done = result.done
|
| 208 |
+
error = obs.query_error
|
| 209 |
+
|
| 210 |
+
rewards.append(reward)
|
| 211 |
+
steps_taken = step
|
| 212 |
+
feedback = obs.feedback
|
| 213 |
+
diagnostics = obs.diagnostics
|
| 214 |
+
|
| 215 |
+
log_step(step=step, action=sql_query, reward=reward, done=done, error=error)
|
| 216 |
+
|
| 217 |
+
if done:
|
| 218 |
+
break
|
| 219 |
+
|
| 220 |
+
score = max(rewards) if rewards else 0.0
|
| 221 |
+
score = min(max(score, 0.0), 1.0)
|
| 222 |
+
success = score >= SUCCESS_THRESHOLD
|
| 223 |
+
|
| 224 |
+
except Exception as e:
|
| 225 |
+
print(f"[DEBUG] Task {task_id} error: {e}", flush=True)
|
| 226 |
+
finally:
|
| 227 |
+
log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
|
| 228 |
+
|
| 229 |
+
return score
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
async def main() -> None:
|
| 233 |
+
"""Run all tasks and print summary."""
|
| 234 |
+
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
|
| 235 |
+
|
| 236 |
+
if IMAGE_NAME:
|
| 237 |
+
env = await SqlQueryEnv.from_docker_image(IMAGE_NAME)
|
| 238 |
+
else:
|
| 239 |
+
base_url = os.getenv("ENV_BASE_URL", "http://localhost:8000")
|
| 240 |
+
env = SqlQueryEnv(base_url=base_url)
|
| 241 |
+
|
| 242 |
+
try:
|
| 243 |
+
async with env:
|
| 244 |
+
scores = []
|
| 245 |
+
for task_id in TASK_IDS:
|
| 246 |
+
score = await run_task(client, env, task_id)
|
| 247 |
+
scores.append(score)
|
| 248 |
+
|
| 249 |
+
# print summary
|
| 250 |
+
print(f"\n{'='*50}", flush=True)
|
| 251 |
+
print(f"[SUMMARY] Model: {MODEL_NAME}", flush=True)
|
| 252 |
+
print(f"[SUMMARY] Average score: {sum(scores)/len(scores):.2f}", flush=True)
|
| 253 |
+
print(f"{'='*50}", flush=True)
|
| 254 |
+
|
| 255 |
+
domains = {"company": [], "hospital": [], "ecommerce": []}
|
| 256 |
+
for tid, sc in zip(TASK_IDS, scores):
|
| 257 |
+
domain = tid.split("_")[0]
|
| 258 |
+
domains[domain].append(sc)
|
| 259 |
+
print(f" {tid}: {sc:.2f}", flush=True)
|
| 260 |
+
|
| 261 |
+
print(f"\nPer-domain averages:", flush=True)
|
| 262 |
+
for domain, dscores in domains.items():
|
| 263 |
+
avg = sum(dscores) / len(dscores) if dscores else 0
|
| 264 |
+
print(f" {domain}: {avg:.2f}", flush=True)
|
| 265 |
+
|
| 266 |
+
except Exception as e:
|
| 267 |
+
print(f"[DEBUG] Environment connection error: {e}", flush=True)
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
if __name__ == "__main__":
|
| 271 |
+
asyncio.run(main())
|
models.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pydantic models for the SQL environment's action/observation space.
|
| 3 |
+
|
| 4 |
+
We tried keeping the observation minimal at first but realized the agent
|
| 5 |
+
needs way more context to actually improve between steps - so we added
|
| 6 |
+
diagnostics, efficiency notes, history etc. Bit of a chonky payload but
|
| 7 |
+
it's all text so doesn't matter much for perf.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from typing import Dict, List, Optional
|
| 11 |
+
|
| 12 |
+
from openenv.core.env_server.types import Action, Observation
|
| 13 |
+
from pydantic import Field
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class SqlQueryAction(Action):
|
| 17 |
+
"""Agent submits a SQL query string."""
|
| 18 |
+
|
| 19 |
+
query: str = Field(
|
| 20 |
+
...,
|
| 21 |
+
description="SQL query to execute against the database. Must be a valid SELECT statement.",
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class SqlQueryObservation(Observation):
|
| 26 |
+
"""
|
| 27 |
+
Full observation returned after each step.
|
| 28 |
+
|
| 29 |
+
We pack a lot in here because the agent needs to understand what went
|
| 30 |
+
wrong and how to fix it. The alternative was returning just a score,
|
| 31 |
+
but that gives basically zero learning signal.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
# task context
|
| 35 |
+
task_id: str = Field(default="", description="Current task identifier")
|
| 36 |
+
difficulty: str = Field(default="easy", description="easy, medium, or hard")
|
| 37 |
+
database_domain: str = Field(
|
| 38 |
+
default="company", description="Which database domain this task uses"
|
| 39 |
+
)
|
| 40 |
+
question: str = Field(default="", description="Natural language question to answer with SQL")
|
| 41 |
+
schema_description: str = Field(
|
| 42 |
+
default="", description="Full database schema with table definitions"
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
# query feedback
|
| 46 |
+
query_result: str = Field(default="", description="Formatted result of the last query")
|
| 47 |
+
query_error: Optional[str] = Field(default=None, description="SQL error message, if any")
|
| 48 |
+
feedback: str = Field(default="", description="Detailed grading feedback")
|
| 49 |
+
|
| 50 |
+
# structured diagnostics - these are the key differentiator
|
| 51 |
+
diagnostics: List[Dict] = Field(
|
| 52 |
+
default_factory=list,
|
| 53 |
+
description="Structured error diagnostics: type, severity, message, suggestion",
|
| 54 |
+
)
|
| 55 |
+
efficiency_notes: List[str] = Field(
|
| 56 |
+
default_factory=list,
|
| 57 |
+
description="SQL best-practice tips based on the submitted query",
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
# hints to help the agent
|
| 61 |
+
expected_row_count: int = Field(default=0, description="Expected number of result rows")
|
| 62 |
+
expected_columns: List[str] = Field(default_factory=list, description="Expected column names")
|
| 63 |
+
|
| 64 |
+
# episode progress
|
| 65 |
+
steps_remaining: int = Field(default=0, description="Attempts left in this episode")
|
| 66 |
+
current_score: float = Field(default=0.0, description="Best score so far (0.0-1.0)")
|
| 67 |
+
history: List[Dict] = Field(default_factory=list, description="Previous queries and scores")
|
openenv.yaml
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
spec_version: 1
|
| 2 |
+
name: sql_query_env
|
| 3 |
+
description: >
|
| 4 |
+
Trains agents to generate SQL queries from natural language questions.
|
| 5 |
+
Tests across three database domains (company analytics, hospital
|
| 6 |
+
management, e-commerce) with efficiency scoring and diagnostic feedback.
|
| 7 |
+
type: space
|
| 8 |
+
runtime: fastapi
|
| 9 |
+
app: server.app:app
|
| 10 |
+
port: 8000
|
| 11 |
+
|
| 12 |
+
tasks:
|
| 13 |
+
# Company
|
| 14 |
+
- id: company_easy_1
|
| 15 |
+
name: "Company β Engineering Employees"
|
| 16 |
+
difficulty: easy
|
| 17 |
+
description: "List active Engineering employees sorted by salary"
|
| 18 |
+
domain: company
|
| 19 |
+
|
| 20 |
+
- id: company_easy_2
|
| 21 |
+
name: "Company β Electronics Products"
|
| 22 |
+
difficulty: easy
|
| 23 |
+
description: "Filter electronics products by price"
|
| 24 |
+
domain: company
|
| 25 |
+
|
| 26 |
+
- id: company_medium_1
|
| 27 |
+
name: "Company β Revenue by Category"
|
| 28 |
+
difficulty: medium
|
| 29 |
+
description: "Total revenue per product category for completed orders"
|
| 30 |
+
domain: company
|
| 31 |
+
|
| 32 |
+
- id: company_medium_2
|
| 33 |
+
name: "Company β Department Salaries"
|
| 34 |
+
difficulty: medium
|
| 35 |
+
description: "Departments with average salary exceeding $100K"
|
| 36 |
+
domain: company
|
| 37 |
+
|
| 38 |
+
- id: company_hard_1
|
| 39 |
+
name: "Company β Budget Utilization"
|
| 40 |
+
difficulty: hard
|
| 41 |
+
description: "Department salary-to-budget utilization percentage"
|
| 42 |
+
domain: company
|
| 43 |
+
|
| 44 |
+
# Hospital
|
| 45 |
+
- id: hospital_easy_1
|
| 46 |
+
name: "Hospital β Current Patients"
|
| 47 |
+
difficulty: easy
|
| 48 |
+
description: "List patients still admitted (NULL discharge date)"
|
| 49 |
+
domain: hospital
|
| 50 |
+
|
| 51 |
+
- id: hospital_medium_1
|
| 52 |
+
name: "Hospital β Medication Costs"
|
| 53 |
+
difficulty: medium
|
| 54 |
+
description: "Total medication cost for admitted patients"
|
| 55 |
+
domain: hospital
|
| 56 |
+
|
| 57 |
+
- id: hospital_hard_1
|
| 58 |
+
name: "Hospital β Cross-Ward Prescriptions"
|
| 59 |
+
difficulty: hard
|
| 60 |
+
description: "Doctors prescribing to patients outside their ward"
|
| 61 |
+
domain: hospital
|
| 62 |
+
|
| 63 |
+
# E-Commerce
|
| 64 |
+
- id: ecommerce_easy_1
|
| 65 |
+
name: "E-Commerce β Premium Users"
|
| 66 |
+
difficulty: easy
|
| 67 |
+
description: "List premium users with signup details"
|
| 68 |
+
domain: ecommerce
|
| 69 |
+
|
| 70 |
+
- id: ecommerce_medium_1
|
| 71 |
+
name: "E-Commerce β Seller Revenue"
|
| 72 |
+
difficulty: medium
|
| 73 |
+
description: "Revenue per seller for delivered orders"
|
| 74 |
+
domain: ecommerce
|
| 75 |
+
|
| 76 |
+
- id: ecommerce_hard_1
|
| 77 |
+
name: "E-Commerce β Return Rate Analysis"
|
| 78 |
+
difficulty: hard
|
| 79 |
+
description: "Return rate per product category with edge cases"
|
| 80 |
+
domain: ecommerce
|
| 81 |
+
|
| 82 |
+
# Expert-level
|
| 83 |
+
- id: company_hard_2
|
| 84 |
+
name: "Company β Salary vs Manager"
|
| 85 |
+
difficulty: hard
|
| 86 |
+
description: "Employees earning more than their direct manager (self-join)"
|
| 87 |
+
domain: company
|
| 88 |
+
|
| 89 |
+
- id: hospital_hard_2
|
| 90 |
+
name: "Hospital β Repeat Visits"
|
| 91 |
+
difficulty: hard
|
| 92 |
+
description: "Patients with multiple appointments with the same doctor"
|
| 93 |
+
domain: hospital
|
| 94 |
+
|
| 95 |
+
- id: ecommerce_hard_2
|
| 96 |
+
name: "E-Commerce β Dead Stock"
|
| 97 |
+
difficulty: hard
|
| 98 |
+
description: "In-stock products that have never been ordered (anti-join)"
|
| 99 |
+
domain: ecommerce
|
| 100 |
+
|
| 101 |
+
action_space:
|
| 102 |
+
type: object
|
| 103 |
+
properties:
|
| 104 |
+
query:
|
| 105 |
+
type: string
|
| 106 |
+
description: "SQL SELECT query to execute against the database"
|
| 107 |
+
|
| 108 |
+
observation_space:
|
| 109 |
+
type: object
|
| 110 |
+
properties:
|
| 111 |
+
task_id:
|
| 112 |
+
type: string
|
| 113 |
+
difficulty:
|
| 114 |
+
type: string
|
| 115 |
+
enum: [easy, medium, hard]
|
| 116 |
+
database_domain:
|
| 117 |
+
type: string
|
| 118 |
+
description: "Company Analytics, Hospital Management, or E-Commerce Platform"
|
| 119 |
+
question:
|
| 120 |
+
type: string
|
| 121 |
+
schema_description:
|
| 122 |
+
type: string
|
| 123 |
+
query_result:
|
| 124 |
+
type: string
|
| 125 |
+
query_error:
|
| 126 |
+
type: string
|
| 127 |
+
feedback:
|
| 128 |
+
type: string
|
| 129 |
+
diagnostics:
|
| 130 |
+
type: array
|
| 131 |
+
description: "Structured diagnostic feedback (type, severity, message, suggestion)"
|
| 132 |
+
efficiency_notes:
|
| 133 |
+
type: array
|
| 134 |
+
description: "SQL efficiency and best-practice feedback"
|
| 135 |
+
expected_row_count:
|
| 136 |
+
type: integer
|
| 137 |
+
expected_columns:
|
| 138 |
+
type: array
|
| 139 |
+
steps_remaining:
|
| 140 |
+
type: integer
|
| 141 |
+
current_score:
|
| 142 |
+
type: number
|
| 143 |
+
history:
|
| 144 |
+
type: array
|
openenv_sql_query_env.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: openenv-sql_query_env
|
| 3 |
+
Version: 0.1.0
|
| 4 |
+
Summary: SQL query generation environment for OpenEnv
|
| 5 |
+
Requires-Python: >=3.10
|
| 6 |
+
License-File: LICENSE
|
| 7 |
+
Requires-Dist: openenv-core[core]>=0.2.2
|
| 8 |
+
Provides-Extra: dev
|
| 9 |
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
| 10 |
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
| 11 |
+
Dynamic: license-file
|
openenv_sql_query_env.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
LICENSE
|
| 2 |
+
README.md
|
| 3 |
+
__init__.py
|
| 4 |
+
client.py
|
| 5 |
+
inference.py
|
| 6 |
+
models.py
|
| 7 |
+
pyproject.toml
|
| 8 |
+
./__init__.py
|
| 9 |
+
./client.py
|
| 10 |
+
./inference.py
|
| 11 |
+
./models.py
|
| 12 |
+
openenv_sql_query_env.egg-info/PKG-INFO
|
| 13 |
+
openenv_sql_query_env.egg-info/SOURCES.txt
|
| 14 |
+
openenv_sql_query_env.egg-info/dependency_links.txt
|
| 15 |
+
openenv_sql_query_env.egg-info/entry_points.txt
|
| 16 |
+
openenv_sql_query_env.egg-info/requires.txt
|
| 17 |
+
openenv_sql_query_env.egg-info/top_level.txt
|
| 18 |
+
server/__init__.py
|
| 19 |
+
server/app.py
|
| 20 |
+
server/graders.py
|
| 21 |
+
server/sql_query_env_environment.py
|
| 22 |
+
server/tasks.py
|
openenv_sql_query_env.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
openenv_sql_query_env.egg-info/entry_points.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[console_scripts]
|
| 2 |
+
server = sql_query_env.server.app:main
|
openenv_sql_query_env.egg-info/requires.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
openenv-core[core]>=0.2.2
|
| 2 |
+
|
| 3 |
+
[dev]
|
| 4 |
+
pytest>=8.0.0
|
| 5 |
+
pytest-cov>=4.0.0
|
openenv_sql_query_env.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
sql_query_env
|
pyproject.toml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=45", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "openenv-sql_query_env"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "SQL query generation environment for OpenEnv"
|
| 9 |
+
requires-python = ">=3.10"
|
| 10 |
+
dependencies = [
|
| 11 |
+
"openenv-core[core]>=0.2.2",
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
[project.optional-dependencies]
|
| 15 |
+
dev = [
|
| 16 |
+
"pytest>=8.0.0",
|
| 17 |
+
"pytest-cov>=4.0.0",
|
| 18 |
+
]
|
| 19 |
+
|
| 20 |
+
[project.scripts]
|
| 21 |
+
server = "sql_query_env.server.app:main"
|
| 22 |
+
|
| 23 |
+
[tool.setuptools]
|
| 24 |
+
include-package-data = true
|
| 25 |
+
packages = ["sql_query_env", "sql_query_env.server"]
|
| 26 |
+
package-dir = { "sql_query_env" = ".", "sql_query_env.server" = "server" }
|
server/Dockerfile
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Multi-stage build using openenv-base
|
| 2 |
+
# This Dockerfile is flexible and works for both:
|
| 3 |
+
# - In-repo environments (with local OpenEnv sources)
|
| 4 |
+
# - Standalone environments (with openenv from PyPI/Git)
|
| 5 |
+
# The build script (openenv build) handles context detection and sets appropriate build args.
|
| 6 |
+
|
| 7 |
+
ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
|
| 8 |
+
FROM ${BASE_IMAGE} AS builder
|
| 9 |
+
|
| 10 |
+
WORKDIR /app
|
| 11 |
+
|
| 12 |
+
# Ensure git is available (required for installing dependencies from VCS)
|
| 13 |
+
RUN apt-get update && \
|
| 14 |
+
apt-get install -y --no-install-recommends git && \
|
| 15 |
+
rm -rf /var/lib/apt/lists/*
|
| 16 |
+
|
| 17 |
+
# Build argument to control whether we're building standalone or in-repo
|
| 18 |
+
ARG BUILD_MODE=in-repo
|
| 19 |
+
ARG ENV_NAME=sql_query_env
|
| 20 |
+
|
| 21 |
+
# Copy environment code (always at root of build context)
|
| 22 |
+
COPY . /app/env
|
| 23 |
+
|
| 24 |
+
# For in-repo builds, openenv is already vendored in the build context
|
| 25 |
+
# For standalone builds, openenv will be installed via pyproject.toml
|
| 26 |
+
WORKDIR /app/env
|
| 27 |
+
|
| 28 |
+
# Ensure uv is available (for local builds where base image lacks it)
|
| 29 |
+
RUN if ! command -v uv >/dev/null 2>&1; then \
|
| 30 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh && \
|
| 31 |
+
mv /root/.local/bin/uv /usr/local/bin/uv && \
|
| 32 |
+
mv /root/.local/bin/uvx /usr/local/bin/uvx; \
|
| 33 |
+
fi
|
| 34 |
+
|
| 35 |
+
# Install dependencies using uv sync
|
| 36 |
+
# If uv.lock exists, use it; otherwise resolve on the fly
|
| 37 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 38 |
+
if [ -f uv.lock ]; then \
|
| 39 |
+
uv sync --frozen --no-install-project --no-editable; \
|
| 40 |
+
else \
|
| 41 |
+
uv sync --no-install-project --no-editable; \
|
| 42 |
+
fi
|
| 43 |
+
|
| 44 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 45 |
+
if [ -f uv.lock ]; then \
|
| 46 |
+
uv sync --frozen --no-editable; \
|
| 47 |
+
else \
|
| 48 |
+
uv sync --no-editable; \
|
| 49 |
+
fi
|
| 50 |
+
|
| 51 |
+
# Final runtime stage
|
| 52 |
+
FROM ${BASE_IMAGE}
|
| 53 |
+
|
| 54 |
+
WORKDIR /app
|
| 55 |
+
|
| 56 |
+
# Copy the virtual environment from builder
|
| 57 |
+
COPY --from=builder /app/env/.venv /app/.venv
|
| 58 |
+
|
| 59 |
+
# Copy the environment code
|
| 60 |
+
COPY --from=builder /app/env /app/env
|
| 61 |
+
|
| 62 |
+
# Set PATH to use the virtual environment
|
| 63 |
+
ENV PATH="/app/.venv/bin:$PATH"
|
| 64 |
+
|
| 65 |
+
# Set PYTHONPATH so imports work correctly
|
| 66 |
+
ENV PYTHONPATH="/app/env:$PYTHONPATH"
|
| 67 |
+
|
| 68 |
+
# Health check
|
| 69 |
+
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 70 |
+
CMD curl -f http://localhost:8000/health || exit 1
|
| 71 |
+
|
| 72 |
+
# Run the FastAPI server
|
| 73 |
+
# The module path is constructed to work with the /app/env structure
|
| 74 |
+
CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]
|
server/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SQL Query Environment Server."""
|
| 2 |
+
|
| 3 |
+
from .sql_query_env_environment import SqlQueryEnvironment
|
| 4 |
+
|
| 5 |
+
__all__ = ["SqlQueryEnvironment"]
|
server/app.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FastAPI server entry point.
|
| 3 |
+
|
| 4 |
+
Start with:
|
| 5 |
+
uv run --project . server
|
| 6 |
+
uvicorn server.app:app --host 0.0.0.0 --port 8000
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
try:
|
| 10 |
+
from openenv.core.env_server.http_server import create_app
|
| 11 |
+
except Exception as e:
|
| 12 |
+
raise ImportError(
|
| 13 |
+
"openenv-core is required. Install with: uv sync"
|
| 14 |
+
) from e
|
| 15 |
+
|
| 16 |
+
# dual-import fallback for different run modes
|
| 17 |
+
try:
|
| 18 |
+
from ..models import SqlQueryAction, SqlQueryObservation
|
| 19 |
+
from .sql_query_env_environment import SqlQueryEnvironment
|
| 20 |
+
except (ImportError, ModuleNotFoundError):
|
| 21 |
+
from models import SqlQueryAction, SqlQueryObservation
|
| 22 |
+
from server.sql_query_env_environment import SqlQueryEnvironment
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
app = create_app(
|
| 26 |
+
SqlQueryEnvironment,
|
| 27 |
+
SqlQueryAction,
|
| 28 |
+
SqlQueryObservation,
|
| 29 |
+
env_name="sql_query_env",
|
| 30 |
+
max_concurrent_envs=256,
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def main():
|
| 35 |
+
"""Entry point for `uv run server` and `openenv serve`."""
|
| 36 |
+
import uvicorn
|
| 37 |
+
uvicorn.run(app, host="0.0.0.0", port=8000)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
if __name__ == "__main__":
|
| 41 |
+
main()
|
server/graders.py
ADDED
|
@@ -0,0 +1,593 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Grading logic for SQL queries.
|
| 3 |
+
|
| 4 |
+
The scoring is split into five components instead of a binary pass/fail.
|
| 5 |
+
We found during testing that a query referencing the right tables but
|
| 6 |
+
messing up column names should still get partial credit β otherwise the
|
| 7 |
+
agent gets zero signal on almost-correct attempts and can't learn.
|
| 8 |
+
|
| 9 |
+
Score weights were chosen by trial and error:
|
| 10 |
+
syntax=0.10, tables=0.15, columns=0.20, results=0.45, efficiency=0.10
|
| 11 |
+
|
| 12 |
+
A query needs to be mostly correct to break 0.70 but you always get
|
| 13 |
+
*something* for partial progress.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import sqlite3
|
| 18 |
+
from dataclasses import dataclass, field
|
| 19 |
+
from typing import Dict, List, Optional, Set, Tuple
|
| 20 |
+
|
| 21 |
+
from .tasks import Task
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
# ---------------------------------------------------------------------------
|
| 25 |
+
# Data structures
|
| 26 |
+
# ---------------------------------------------------------------------------
|
| 27 |
+
|
| 28 |
+
@dataclass
|
| 29 |
+
class Diagnostic:
|
| 30 |
+
"""Structured diagnostic for process supervision feedback."""
|
| 31 |
+
type: str # e.g. MISSING_JOIN, WRONG_TABLE
|
| 32 |
+
severity: str # error / warning / info
|
| 33 |
+
message: str
|
| 34 |
+
suggestion: str
|
| 35 |
+
|
| 36 |
+
def to_dict(self) -> Dict:
|
| 37 |
+
return {
|
| 38 |
+
"type": self.type,
|
| 39 |
+
"severity": self.severity,
|
| 40 |
+
"message": self.message,
|
| 41 |
+
"suggestion": self.suggestion,
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class GradeResult:
|
| 47 |
+
"""Holds all scoring info for a single query attempt."""
|
| 48 |
+
|
| 49 |
+
total_score: float
|
| 50 |
+
syntax_score: float
|
| 51 |
+
table_score: float
|
| 52 |
+
column_score: float
|
| 53 |
+
result_score: float
|
| 54 |
+
efficiency_score: float
|
| 55 |
+
penalty: float
|
| 56 |
+
feedback: str
|
| 57 |
+
query_result: Optional[str]
|
| 58 |
+
error: Optional[str]
|
| 59 |
+
rows_returned: int
|
| 60 |
+
rows_expected: int
|
| 61 |
+
diagnostics: List[Diagnostic] = field(default_factory=list)
|
| 62 |
+
efficiency_notes: List[str] = field(default_factory=list)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
# ---------------------------------------------------------------------------
|
| 66 |
+
# Helper functions
|
| 67 |
+
# ---------------------------------------------------------------------------
|
| 68 |
+
|
| 69 |
+
def _extract_tables(query: str) -> Set[str]:
|
| 70 |
+
"""Pull table names out of FROM and JOIN clauses.
|
| 71 |
+
|
| 72 |
+
Quick regex approach β doesn't handle subquery aliases or CTEs but
|
| 73 |
+
it's good enough for our task set. Didn't want to pull in sqlparse
|
| 74 |
+
just for this.
|
| 75 |
+
"""
|
| 76 |
+
q = " ".join(query.lower().split())
|
| 77 |
+
tables = set()
|
| 78 |
+
for match in re.finditer(r'\bfrom\s+(\w+)', q):
|
| 79 |
+
tables.add(match.group(1))
|
| 80 |
+
for match in re.finditer(r'\bjoin\s+(\w+)', q):
|
| 81 |
+
tables.add(match.group(1))
|
| 82 |
+
return tables
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _format_result(rows: List[Tuple], columns: List[str], max_rows: int = 20) -> str:
|
| 86 |
+
"""Pretty-print query results."""
|
| 87 |
+
if not rows:
|
| 88 |
+
return "(no rows returned)"
|
| 89 |
+
|
| 90 |
+
lines = []
|
| 91 |
+
header = " | ".join(columns)
|
| 92 |
+
lines.append(header)
|
| 93 |
+
lines.append("-" * len(header))
|
| 94 |
+
|
| 95 |
+
for row in rows[:max_rows]:
|
| 96 |
+
lines.append(" | ".join(str(v) for v in row))
|
| 97 |
+
if len(rows) > max_rows:
|
| 98 |
+
lines.append(f"... ({len(rows) - max_rows} more rows)")
|
| 99 |
+
|
| 100 |
+
return "\n".join(lines)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _is_destructive(query: str) -> bool:
|
| 104 |
+
"""Check if query tries to modify data. We block everything except SELECT.
|
| 105 |
+
|
| 106 |
+
Double-checking here even though the environment also uses read-only
|
| 107 |
+
connections β better safe than sorry.
|
| 108 |
+
"""
|
| 109 |
+
q = query.upper().strip()
|
| 110 |
+
bad_keywords = ["DROP", "DELETE", "TRUNCATE", "ALTER", "UPDATE", "INSERT", "CREATE"]
|
| 111 |
+
for kw in bad_keywords:
|
| 112 |
+
if re.search(rf'\b{kw}\b', q):
|
| 113 |
+
return True
|
| 114 |
+
return False
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _normalize_value(v) -> str:
|
| 118 |
+
"""Normalize cell values for comparison.
|
| 119 |
+
|
| 120 |
+
Rounds floats to 2 decimal places so 3.1000001 matches 3.10.
|
| 121 |
+
"""
|
| 122 |
+
if v is None:
|
| 123 |
+
return "NULL"
|
| 124 |
+
if isinstance(v, float):
|
| 125 |
+
return f"{v:.2f}"
|
| 126 |
+
return str(v).strip().lower()
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _compare_result_sets(
|
| 130 |
+
actual_rows: List[Tuple],
|
| 131 |
+
expected_rows: List[Tuple],
|
| 132 |
+
query: str = "",
|
| 133 |
+
ground_truth_query: str = "",
|
| 134 |
+
) -> float:
|
| 135 |
+
"""Compare actual vs expected results, returns similarity 0-1.
|
| 136 |
+
|
| 137 |
+
Also checks row ordering if the ground truth has ORDER BY.
|
| 138 |
+
"""
|
| 139 |
+
if not expected_rows:
|
| 140 |
+
return 1.0 if not actual_rows else 0.0
|
| 141 |
+
if not actual_rows:
|
| 142 |
+
return 0.0
|
| 143 |
+
|
| 144 |
+
def normalize_row(row: Tuple) -> Tuple[str, ...]:
|
| 145 |
+
return tuple(_normalize_value(v) for v in row)
|
| 146 |
+
|
| 147 |
+
actual_norm = [normalize_row(r) for r in actual_rows]
|
| 148 |
+
expected_norm = [normalize_row(r) for r in expected_rows]
|
| 149 |
+
|
| 150 |
+
actual_set = set(actual_norm)
|
| 151 |
+
expected_set = set(expected_norm)
|
| 152 |
+
|
| 153 |
+
matching = actual_set & expected_set
|
| 154 |
+
total_expected = len(expected_set)
|
| 155 |
+
|
| 156 |
+
if total_expected == 0:
|
| 157 |
+
return 1.0
|
| 158 |
+
|
| 159 |
+
# what fraction of expected rows did we get?
|
| 160 |
+
content_score = len(matching) / total_expected
|
| 161 |
+
|
| 162 |
+
# dock points for extra rows the agent shouldn't have returned
|
| 163 |
+
extra = len(actual_set - expected_set)
|
| 164 |
+
if extra > 0:
|
| 165 |
+
content_score = max(0.0, content_score - min(extra / total_expected * 0.3, 0.3))
|
| 166 |
+
|
| 167 |
+
# bonus for correct ordering when ORDER BY is expected
|
| 168 |
+
order_bonus = 0.0
|
| 169 |
+
has_order = "ORDER BY" in ground_truth_query.upper()
|
| 170 |
+
if has_order and content_score > 0.5 and len(actual_norm) == len(expected_norm):
|
| 171 |
+
correct_positions = sum(
|
| 172 |
+
1 for a, e in zip(actual_norm, expected_norm) if a == e
|
| 173 |
+
)
|
| 174 |
+
order_bonus = (correct_positions / len(expected_norm)) * 0.1
|
| 175 |
+
|
| 176 |
+
return min(content_score + order_bonus, 1.0)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# ---------------------------------------------------------------------------
|
| 180 |
+
# Diagnostic engine
|
| 181 |
+
# ---------------------------------------------------------------------------
|
| 182 |
+
|
| 183 |
+
def _diagnose_query(
|
| 184 |
+
query: str,
|
| 185 |
+
task: Task,
|
| 186 |
+
actual_rows: List[Tuple],
|
| 187 |
+
expected_rows: List[Tuple],
|
| 188 |
+
actual_cols: List[str],
|
| 189 |
+
expected_cols: List[str],
|
| 190 |
+
error: Optional[str],
|
| 191 |
+
) -> List[Diagnostic]:
|
| 192 |
+
"""Figure out what went wrong and give actionable feedback.
|
| 193 |
+
|
| 194 |
+
We check for ~17 different issue types. The goal is process
|
| 195 |
+
supervision: give the agent enough info to fix its query on
|
| 196 |
+
the next attempt.
|
| 197 |
+
"""
|
| 198 |
+
diagnostics = []
|
| 199 |
+
q_upper = query.upper()
|
| 200 |
+
gt_upper = task.ground_truth_query.upper()
|
| 201 |
+
|
| 202 |
+
# -- syntax errors (parsed from SQLite error messages) --
|
| 203 |
+
if error:
|
| 204 |
+
if "no such table" in error.lower():
|
| 205 |
+
table_match = re.search(r'no such table: (\w+)', error, re.IGNORECASE)
|
| 206 |
+
table_name = table_match.group(1) if table_match else "unknown"
|
| 207 |
+
diagnostics.append(Diagnostic(
|
| 208 |
+
type="WRONG_TABLE_NAME",
|
| 209 |
+
severity="error",
|
| 210 |
+
message=f"Table '{table_name}' does not exist in this database.",
|
| 211 |
+
suggestion="Check the schema description for available table names.",
|
| 212 |
+
))
|
| 213 |
+
elif "no such column" in error.lower():
|
| 214 |
+
col_match = re.search(r'no such column: (\S+)', error, re.IGNORECASE)
|
| 215 |
+
col_name = col_match.group(1) if col_match else "unknown"
|
| 216 |
+
diagnostics.append(Diagnostic(
|
| 217 |
+
type="WRONG_COLUMN_NAME",
|
| 218 |
+
severity="error",
|
| 219 |
+
message=f"Column '{col_name}' does not exist.",
|
| 220 |
+
suggestion="Check the table definition for correct column names.",
|
| 221 |
+
))
|
| 222 |
+
elif "ambiguous column" in error.lower():
|
| 223 |
+
diagnostics.append(Diagnostic(
|
| 224 |
+
type="AMBIGUOUS_COLUMN",
|
| 225 |
+
severity="error",
|
| 226 |
+
message="A column name is ambiguous - it exists in multiple tables.",
|
| 227 |
+
suggestion="Use table aliases (e.g. e.name instead of just name).",
|
| 228 |
+
))
|
| 229 |
+
else:
|
| 230 |
+
diagnostics.append(Diagnostic(
|
| 231 |
+
type="SYNTAX_ERROR",
|
| 232 |
+
severity="error",
|
| 233 |
+
message=f"SQL syntax error: {error[:100]}",
|
| 234 |
+
suggestion="Check your SQL syntax. Common issues: missing commas, unmatched parentheses, typos.",
|
| 235 |
+
))
|
| 236 |
+
return diagnostics
|
| 237 |
+
|
| 238 |
+
# -- structural checks --
|
| 239 |
+
expected_tables = _extract_tables(task.ground_truth_query)
|
| 240 |
+
actual_tables = _extract_tables(query)
|
| 241 |
+
|
| 242 |
+
missing_tables = expected_tables - actual_tables
|
| 243 |
+
if missing_tables:
|
| 244 |
+
diagnostics.append(Diagnostic(
|
| 245 |
+
type="MISSING_TABLE",
|
| 246 |
+
severity="warning",
|
| 247 |
+
message=f"Your query doesn't reference: {', '.join(missing_tables)}",
|
| 248 |
+
suggestion=f"You probably need to JOIN with {', '.join(missing_tables)} to get the required data.",
|
| 249 |
+
))
|
| 250 |
+
|
| 251 |
+
if "JOIN" in gt_upper and "JOIN" not in q_upper:
|
| 252 |
+
diagnostics.append(Diagnostic(
|
| 253 |
+
type="MISSING_JOIN",
|
| 254 |
+
severity="warning",
|
| 255 |
+
message="This question requires data from multiple tables, but no JOIN was found.",
|
| 256 |
+
suggestion="Use JOIN to combine tables. Example: SELECT ... FROM t1 JOIN t2 ON t1.id = t2.t1_id",
|
| 257 |
+
))
|
| 258 |
+
|
| 259 |
+
if "GROUP BY" in gt_upper and "GROUP BY" not in q_upper:
|
| 260 |
+
diagnostics.append(Diagnostic(
|
| 261 |
+
type="MISSING_AGGREGATION",
|
| 262 |
+
severity="warning",
|
| 263 |
+
message="This question requires grouping/aggregation, but no GROUP BY was found.",
|
| 264 |
+
suggestion="Use GROUP BY with aggregate functions (SUM, COUNT, AVG) to group results.",
|
| 265 |
+
))
|
| 266 |
+
|
| 267 |
+
if "HAVING" in gt_upper and "HAVING" not in q_upper and "GROUP BY" in q_upper:
|
| 268 |
+
diagnostics.append(Diagnostic(
|
| 269 |
+
type="MISSING_HAVING",
|
| 270 |
+
severity="info",
|
| 271 |
+
message="You might need a HAVING clause to filter grouped results.",
|
| 272 |
+
suggestion="HAVING filters after GROUP BY. Example: HAVING COUNT(*) > 5",
|
| 273 |
+
))
|
| 274 |
+
|
| 275 |
+
if "IS NULL" in gt_upper and "IS NULL" not in q_upper:
|
| 276 |
+
diagnostics.append(Diagnostic(
|
| 277 |
+
type="NULL_HANDLING",
|
| 278 |
+
severity="warning",
|
| 279 |
+
message="This question involves NULL values. Use IS NULL or IS NOT NULL.",
|
| 280 |
+
suggestion="In SQL, NULL != NULL. Use IS NULL to check for missing values, not = NULL.",
|
| 281 |
+
))
|
| 282 |
+
|
| 283 |
+
# -- result set checks --
|
| 284 |
+
missing_cols = set(expected_cols) - set(actual_cols)
|
| 285 |
+
if missing_cols:
|
| 286 |
+
diagnostics.append(Diagnostic(
|
| 287 |
+
type="MISSING_COLUMNS",
|
| 288 |
+
severity="warning",
|
| 289 |
+
message=f"Missing output columns: {', '.join(missing_cols)}",
|
| 290 |
+
suggestion=f"Add these to your SELECT: {', '.join(missing_cols)}",
|
| 291 |
+
))
|
| 292 |
+
|
| 293 |
+
extra_cols = set(actual_cols) - set(expected_cols)
|
| 294 |
+
if extra_cols:
|
| 295 |
+
diagnostics.append(Diagnostic(
|
| 296 |
+
type="EXTRA_COLUMNS",
|
| 297 |
+
severity="info",
|
| 298 |
+
message=f"Extra columns in output: {', '.join(extra_cols)}",
|
| 299 |
+
suggestion=f"Expected columns are: {', '.join(expected_cols)}",
|
| 300 |
+
))
|
| 301 |
+
|
| 302 |
+
if len(actual_rows) > len(expected_rows):
|
| 303 |
+
diagnostics.append(Diagnostic(
|
| 304 |
+
type="EXTRA_ROWS",
|
| 305 |
+
severity="warning",
|
| 306 |
+
message=f"Got {len(actual_rows)} rows but expected {len(expected_rows)}.",
|
| 307 |
+
suggestion="Your WHERE/HAVING conditions may be too broad. Try adding more filters.",
|
| 308 |
+
))
|
| 309 |
+
elif len(actual_rows) < len(expected_rows) and len(actual_rows) > 0:
|
| 310 |
+
diagnostics.append(Diagnostic(
|
| 311 |
+
type="MISSING_ROWS",
|
| 312 |
+
severity="warning",
|
| 313 |
+
message=f"Got {len(actual_rows)} rows but expected {len(expected_rows)}.",
|
| 314 |
+
suggestion="Filters might be too restrictive, or try LEFT JOIN instead of INNER JOIN.",
|
| 315 |
+
))
|
| 316 |
+
|
| 317 |
+
return diagnostics
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
# ---------------------------------------------------------------------------
|
| 321 |
+
# Efficiency scoring
|
| 322 |
+
# ---------------------------------------------------------------------------
|
| 323 |
+
|
| 324 |
+
def _score_efficiency(
|
| 325 |
+
query: str,
|
| 326 |
+
task: Task,
|
| 327 |
+
actual_cols: List[str],
|
| 328 |
+
expected_cols: List[str],
|
| 329 |
+
) -> Tuple[float, List[str]]:
|
| 330 |
+
"""Score SQL best practices (0.0 - 0.10).
|
| 331 |
+
|
| 332 |
+
Rewards good habits and penalizes bad ones. The weights are small
|
| 333 |
+
(max 0.10 total) so this doesn't dominate scoring, but it adds up
|
| 334 |
+
and teaches the agent to write cleaner SQL.
|
| 335 |
+
"""
|
| 336 |
+
notes = []
|
| 337 |
+
score = 0.0
|
| 338 |
+
q_upper = query.upper().strip()
|
| 339 |
+
|
| 340 |
+
# -- good practices --
|
| 341 |
+
if re.search(r'\bAS\s+\w+', query, re.IGNORECASE):
|
| 342 |
+
score += 0.03
|
| 343 |
+
notes.append("Good: using column aliases (AS)")
|
| 344 |
+
|
| 345 |
+
if "SELECT *" not in q_upper and len(expected_cols) > 0:
|
| 346 |
+
if set(actual_cols) == set(expected_cols) or set(actual_cols).issuperset(set(expected_cols)):
|
| 347 |
+
score += 0.02
|
| 348 |
+
notes.append("Good: selecting specific columns instead of SELECT *")
|
| 349 |
+
|
| 350 |
+
alias_pattern = r'\b(?:FROM|JOIN)\s+\w+\s+(?:AS\s+)?([a-z]\w{0,2})\b'
|
| 351 |
+
if re.search(alias_pattern, query, re.IGNORECASE):
|
| 352 |
+
score += 0.02
|
| 353 |
+
notes.append("Good: using table aliases for readability")
|
| 354 |
+
|
| 355 |
+
if re.search(r'\bCOALESCE\b|\bIFNULL\b', query, re.IGNORECASE):
|
| 356 |
+
score += 0.03
|
| 357 |
+
notes.append("Good: proper NULL handling with COALESCE/IFNULL")
|
| 358 |
+
|
| 359 |
+
# -- bad practices --
|
| 360 |
+
if "SELECT *" in q_upper and len(expected_cols) > 0 and len(expected_cols) < 8:
|
| 361 |
+
score -= 0.02
|
| 362 |
+
notes.append("Avoid SELECT * when you only need specific columns")
|
| 363 |
+
|
| 364 |
+
if "DISTINCT" in q_upper and "DISTINCT" not in task.ground_truth_query.upper():
|
| 365 |
+
score -= 0.01
|
| 366 |
+
notes.append("DISTINCT may not be needed here - check if duplicates are actually expected")
|
| 367 |
+
|
| 368 |
+
# check for cartesian product (FROM a, b without JOIN)
|
| 369 |
+
from_tables = re.findall(r'\bFROM\s+(\w+(?:\s*,\s*\w+)+)', query, re.IGNORECASE)
|
| 370 |
+
if from_tables and "JOIN" not in q_upper:
|
| 371 |
+
score -= 0.02
|
| 372 |
+
notes.append("Warning: potential Cartesian product - use explicit JOIN instead of comma-separated tables")
|
| 373 |
+
|
| 374 |
+
score = max(0.0, min(score, 0.10))
|
| 375 |
+
return score, notes
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
# ---------------------------------------------------------------------------
|
| 379 |
+
# Main grading function
|
| 380 |
+
# ---------------------------------------------------------------------------
|
| 381 |
+
|
| 382 |
+
def grade_query(
|
| 383 |
+
conn: sqlite3.Connection,
|
| 384 |
+
query: str,
|
| 385 |
+
task: Task,
|
| 386 |
+
previous_queries: List[str],
|
| 387 |
+
) -> GradeResult:
|
| 388 |
+
"""Grade a submitted SQL query.
|
| 389 |
+
|
| 390 |
+
Scoring breakdown:
|
| 391 |
+
syntax: 0.10 (query runs without error)
|
| 392 |
+
tables: 0.15 (correct tables referenced)
|
| 393 |
+
columns: 0.20 (correct output columns)
|
| 394 |
+
results: 0.45 (correct data returned)
|
| 395 |
+
efficiency: 0.10 (query quality)
|
| 396 |
+
penalties: -0.10 for destructive SQL, -0.05 for repeated queries
|
| 397 |
+
"""
|
| 398 |
+
feedback_parts = []
|
| 399 |
+
syntax_score = 0.0
|
| 400 |
+
table_score = 0.0
|
| 401 |
+
column_score = 0.0
|
| 402 |
+
result_score = 0.0
|
| 403 |
+
efficiency_score = 0.0
|
| 404 |
+
penalty = 0.0
|
| 405 |
+
query_result_str = None
|
| 406 |
+
error_msg = None
|
| 407 |
+
rows_returned = 0
|
| 408 |
+
rows_expected = 0
|
| 409 |
+
diagnostics = []
|
| 410 |
+
efficiency_notes = []
|
| 411 |
+
|
| 412 |
+
# check for destructive queries
|
| 413 |
+
if _is_destructive(query):
|
| 414 |
+
penalty -= 0.10
|
| 415 |
+
feedback_parts.append("PENALTY: Destructive SQL detected. Only SELECT statements are allowed.")
|
| 416 |
+
diagnostics.append(Diagnostic(
|
| 417 |
+
type="DESTRUCTIVE_QUERY",
|
| 418 |
+
severity="error",
|
| 419 |
+
message="Destructive operations (DROP, DELETE, etc.) are not allowed.",
|
| 420 |
+
suggestion="Use only SELECT statements to query data.",
|
| 421 |
+
))
|
| 422 |
+
|
| 423 |
+
# check for repeated query
|
| 424 |
+
normalized_query = " ".join(query.lower().split())
|
| 425 |
+
for prev in previous_queries:
|
| 426 |
+
if " ".join(prev.lower().split()) == normalized_query:
|
| 427 |
+
penalty -= 0.05
|
| 428 |
+
feedback_parts.append("PENALTY: Identical query submitted before. Try something different.")
|
| 429 |
+
diagnostics.append(Diagnostic(
|
| 430 |
+
type="REPEATED_QUERY",
|
| 431 |
+
severity="warning",
|
| 432 |
+
message="This exact query was already submitted.",
|
| 433 |
+
suggestion="Modify your query based on the previous feedback to improve.",
|
| 434 |
+
))
|
| 435 |
+
break
|
| 436 |
+
|
| 437 |
+
# 1. Syntax check (0.10)
|
| 438 |
+
actual_rows = []
|
| 439 |
+
actual_cols = []
|
| 440 |
+
try:
|
| 441 |
+
cursor = conn.execute(query)
|
| 442 |
+
actual_cols = [desc[0].lower() for desc in cursor.description] if cursor.description else []
|
| 443 |
+
actual_rows = cursor.fetchall()
|
| 444 |
+
rows_returned = len(actual_rows)
|
| 445 |
+
syntax_score = 0.10
|
| 446 |
+
feedback_parts.append("Query executed successfully.")
|
| 447 |
+
query_result_str = _format_result(actual_rows, actual_cols)
|
| 448 |
+
except Exception as e:
|
| 449 |
+
error_msg = str(e)
|
| 450 |
+
feedback_parts.append(f"SQL Error: {error_msg}")
|
| 451 |
+
|
| 452 |
+
diagnostics = _diagnose_query(
|
| 453 |
+
query, task, [], [], [], [], error_msg
|
| 454 |
+
)
|
| 455 |
+
|
| 456 |
+
total = max(0.0, penalty)
|
| 457 |
+
return GradeResult(
|
| 458 |
+
total_score=total,
|
| 459 |
+
syntax_score=0.0,
|
| 460 |
+
table_score=0.0,
|
| 461 |
+
column_score=0.0,
|
| 462 |
+
result_score=0.0,
|
| 463 |
+
efficiency_score=0.0,
|
| 464 |
+
penalty=penalty,
|
| 465 |
+
feedback="\n".join(feedback_parts),
|
| 466 |
+
query_result=None,
|
| 467 |
+
error=error_msg,
|
| 468 |
+
rows_returned=0,
|
| 469 |
+
rows_expected=0,
|
| 470 |
+
diagnostics=diagnostics,
|
| 471 |
+
efficiency_notes=[],
|
| 472 |
+
)
|
| 473 |
+
|
| 474 |
+
# get ground truth results
|
| 475 |
+
try:
|
| 476 |
+
expected_cursor = conn.execute(task.ground_truth_query)
|
| 477 |
+
expected_rows = expected_cursor.fetchall()
|
| 478 |
+
expected_cols_from_query = [
|
| 479 |
+
desc[0].lower() for desc in expected_cursor.description
|
| 480 |
+
] if expected_cursor.description else []
|
| 481 |
+
rows_expected = len(expected_rows)
|
| 482 |
+
except Exception:
|
| 483 |
+
expected_rows = []
|
| 484 |
+
expected_cols_from_query = []
|
| 485 |
+
rows_expected = 0
|
| 486 |
+
|
| 487 |
+
# 2. Table references (0.15)
|
| 488 |
+
expected_tables = _extract_tables(task.ground_truth_query)
|
| 489 |
+
actual_tables = _extract_tables(query)
|
| 490 |
+
|
| 491 |
+
if expected_tables:
|
| 492 |
+
matching_tables = actual_tables & expected_tables
|
| 493 |
+
table_ratio = len(matching_tables) / len(expected_tables)
|
| 494 |
+
table_score = 0.15 * table_ratio
|
| 495 |
+
|
| 496 |
+
if table_ratio == 1.0:
|
| 497 |
+
feedback_parts.append("Correct tables referenced.")
|
| 498 |
+
elif table_ratio > 0:
|
| 499 |
+
missing = expected_tables - actual_tables
|
| 500 |
+
feedback_parts.append(f"Partially correct tables. Missing: {', '.join(missing)}")
|
| 501 |
+
else:
|
| 502 |
+
feedback_parts.append(f"Wrong tables. Consider using: {', '.join(expected_tables)}")
|
| 503 |
+
|
| 504 |
+
# 3. Column check (0.20)
|
| 505 |
+
expected_cols_lower = [c.lower() for c in task.expected_columns]
|
| 506 |
+
|
| 507 |
+
if expected_cols_lower:
|
| 508 |
+
matching_cols = set(actual_cols) & set(expected_cols_lower)
|
| 509 |
+
col_ratio = len(matching_cols) / len(expected_cols_lower)
|
| 510 |
+
column_score = 0.20 * col_ratio
|
| 511 |
+
|
| 512 |
+
if col_ratio == 1.0:
|
| 513 |
+
feedback_parts.append("Correct output columns.")
|
| 514 |
+
elif col_ratio > 0:
|
| 515 |
+
missing_cols = set(expected_cols_lower) - set(actual_cols)
|
| 516 |
+
feedback_parts.append(f"Missing columns: {', '.join(missing_cols)}")
|
| 517 |
+
else:
|
| 518 |
+
feedback_parts.append(
|
| 519 |
+
f"Wrong columns. Expected: {', '.join(expected_cols_lower)}. Got: {', '.join(actual_cols)}"
|
| 520 |
+
)
|
| 521 |
+
|
| 522 |
+
# 4. Result set comparison (0.45)
|
| 523 |
+
similarity = _compare_result_sets(
|
| 524 |
+
actual_rows, expected_rows, query, task.ground_truth_query
|
| 525 |
+
)
|
| 526 |
+
result_score = 0.45 * similarity
|
| 527 |
+
|
| 528 |
+
if similarity == 1.0:
|
| 529 |
+
feedback_parts.append("Perfect result set - all rows match.")
|
| 530 |
+
elif similarity > 0.5:
|
| 531 |
+
feedback_parts.append(
|
| 532 |
+
f"Partial match: {similarity:.0%} of expected rows. "
|
| 533 |
+
f"Got {rows_returned} rows, expected {rows_expected}."
|
| 534 |
+
)
|
| 535 |
+
elif similarity > 0:
|
| 536 |
+
feedback_parts.append(
|
| 537 |
+
f"Low match: {similarity:.0%} of expected rows. "
|
| 538 |
+
f"Got {rows_returned} rows, expected {rows_expected}."
|
| 539 |
+
)
|
| 540 |
+
else:
|
| 541 |
+
feedback_parts.append(
|
| 542 |
+
f"No matching rows. Expected {rows_expected} rows, got {rows_returned}."
|
| 543 |
+
)
|
| 544 |
+
|
| 545 |
+
# 5. Efficiency (0.10)
|
| 546 |
+
efficiency_score, efficiency_notes = _score_efficiency(
|
| 547 |
+
query, task, actual_cols, expected_cols_lower
|
| 548 |
+
)
|
| 549 |
+
|
| 550 |
+
# generate diagnostics
|
| 551 |
+
diagnostics = _diagnose_query(
|
| 552 |
+
query, task, actual_rows, expected_rows,
|
| 553 |
+
actual_cols, expected_cols_lower, None,
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
# compute final score
|
| 557 |
+
total = syntax_score + table_score + column_score + result_score + efficiency_score + penalty
|
| 558 |
+
total = max(0.0, min(total, 1.0))
|
| 559 |
+
|
| 560 |
+
# score breakdown
|
| 561 |
+
feedback_parts.append(
|
| 562 |
+
f"\nScore: syntax={syntax_score:.2f} + tables={table_score:.2f} + "
|
| 563 |
+
f"columns={column_score:.2f} + results={result_score:.2f} + "
|
| 564 |
+
f"efficiency={efficiency_score:.2f} + penalties={penalty:.2f} = {total:.2f}"
|
| 565 |
+
)
|
| 566 |
+
|
| 567 |
+
if efficiency_notes:
|
| 568 |
+
feedback_parts.append("\nQuery quality notes:")
|
| 569 |
+
feedback_parts.extend(f" - {note}" for note in efficiency_notes)
|
| 570 |
+
|
| 571 |
+
if diagnostics:
|
| 572 |
+
feedback_parts.append("\nIssues found:")
|
| 573 |
+
for d in diagnostics:
|
| 574 |
+
severity_tag = "ERR" if d.severity == "error" else "WARN" if d.severity == "warning" else "INFO"
|
| 575 |
+
feedback_parts.append(f" [{severity_tag}] {d.type}: {d.message}")
|
| 576 |
+
feedback_parts.append(f" Suggestion: {d.suggestion}")
|
| 577 |
+
|
| 578 |
+
return GradeResult(
|
| 579 |
+
total_score=total,
|
| 580 |
+
syntax_score=syntax_score,
|
| 581 |
+
table_score=table_score,
|
| 582 |
+
column_score=column_score,
|
| 583 |
+
result_score=result_score,
|
| 584 |
+
efficiency_score=efficiency_score,
|
| 585 |
+
penalty=penalty,
|
| 586 |
+
feedback="\n".join(feedback_parts),
|
| 587 |
+
query_result=query_result_str,
|
| 588 |
+
error=error_msg,
|
| 589 |
+
rows_returned=rows_returned,
|
| 590 |
+
rows_expected=rows_expected,
|
| 591 |
+
diagnostics=diagnostics,
|
| 592 |
+
efficiency_notes=efficiency_notes,
|
| 593 |
+
)
|
server/requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
openenv[core]>=0.2.0
|
| 2 |
+
fastapi>=0.115.0
|
| 3 |
+
uvicorn>=0.24.0
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
|
server/sql_query_env_environment.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Core environment logic for the SQL query environment.
|
| 3 |
+
|
| 4 |
+
Each episode creates a fresh in-memory SQLite database for the chosen
|
| 5 |
+
domain so there's no state leaking between episodes. SQLite in-memory
|
| 6 |
+
databases are cheap to spin up so this doesn't hurt performance.
|
| 7 |
+
|
| 8 |
+
TODO(team): might want to add a "curriculum mode" that auto-selects
|
| 9 |
+
tasks of increasing difficulty within a session. For now we just let
|
| 10 |
+
the caller pick or we cycle through randomly.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import random
|
| 14 |
+
import sqlite3
|
| 15 |
+
from typing import Dict, List, Optional
|
| 16 |
+
from uuid import uuid4
|
| 17 |
+
|
| 18 |
+
from openenv.core.env_server.interfaces import Environment
|
| 19 |
+
from openenv.core.env_server.types import State
|
| 20 |
+
|
| 21 |
+
# Handle both import paths β running via `uv run server` uses a different
|
| 22 |
+
# package layout than `python -m sql_query_env.server.app`
|
| 23 |
+
try:
|
| 24 |
+
from ..models import SqlQueryAction, SqlQueryObservation
|
| 25 |
+
except ImportError:
|
| 26 |
+
from models import SqlQueryAction, SqlQueryObservation
|
| 27 |
+
|
| 28 |
+
try:
|
| 29 |
+
from .tasks import (
|
| 30 |
+
ALL_TASKS,
|
| 31 |
+
ALL_TASKS_LIST,
|
| 32 |
+
SCHEMAS,
|
| 33 |
+
TASK_MAP,
|
| 34 |
+
Task,
|
| 35 |
+
create_database,
|
| 36 |
+
get_expected_result,
|
| 37 |
+
)
|
| 38 |
+
from .graders import grade_query, GradeResult
|
| 39 |
+
except ImportError:
|
| 40 |
+
from server.tasks import (
|
| 41 |
+
ALL_TASKS,
|
| 42 |
+
ALL_TASKS_LIST,
|
| 43 |
+
SCHEMAS,
|
| 44 |
+
TASK_MAP,
|
| 45 |
+
Task,
|
| 46 |
+
create_database,
|
| 47 |
+
get_expected_result,
|
| 48 |
+
)
|
| 49 |
+
from server.graders import grade_query, GradeResult
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class SqlQueryEnvironment(Environment):
|
| 53 |
+
"""Main environment class - presents schema + question, grades SQL.
|
| 54 |
+
|
| 55 |
+
Steps are independent in terms of DB state (no writes accumulate).
|
| 56 |
+
We track query history and best score across steps for penalty logic
|
| 57 |
+
and progressive hints.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
SUPPORTS_CONCURRENT_SESSIONS: bool = True
|
| 61 |
+
|
| 62 |
+
def __init__(self):
|
| 63 |
+
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 64 |
+
self._db: Optional[sqlite3.Connection] = None
|
| 65 |
+
self._current_task: Optional[Task] = None
|
| 66 |
+
self._current_schema_id: Optional[str] = None
|
| 67 |
+
self._previous_queries: List[str] = []
|
| 68 |
+
self._best_score: float = 0.0
|
| 69 |
+
self._step_rewards: List[float] = []
|
| 70 |
+
self._task_queue: List[str] = []
|
| 71 |
+
self._current_task_index: int = 0
|
| 72 |
+
|
| 73 |
+
def _get_task_list(self) -> List[str]:
|
| 74 |
+
"""Shuffle task IDs for random ordering."""
|
| 75 |
+
all_ids = [t.id for t in ALL_TASKS_LIST]
|
| 76 |
+
random.shuffle(all_ids)
|
| 77 |
+
return all_ids
|
| 78 |
+
|
| 79 |
+
def _select_task(self, task_id: Optional[str] = None) -> Task:
|
| 80 |
+
"""Pick a specific task or grab the next one from the queue."""
|
| 81 |
+
if task_id and task_id in TASK_MAP:
|
| 82 |
+
return TASK_MAP[task_id]
|
| 83 |
+
|
| 84 |
+
# lazy-init the queue
|
| 85 |
+
if not self._task_queue:
|
| 86 |
+
self._task_queue = self._get_task_list()
|
| 87 |
+
self._current_task_index = 0
|
| 88 |
+
|
| 89 |
+
if self._current_task_index >= len(self._task_queue):
|
| 90 |
+
self._current_task_index = 0
|
| 91 |
+
|
| 92 |
+
task = TASK_MAP[self._task_queue[self._current_task_index]]
|
| 93 |
+
self._current_task_index += 1
|
| 94 |
+
return task
|
| 95 |
+
|
| 96 |
+
def reset(self, task_id: Optional[str] = None) -> SqlQueryObservation:
|
| 97 |
+
"""Start a new episode.
|
| 98 |
+
|
| 99 |
+
Creates a fresh DB for the task's domain and returns the initial
|
| 100 |
+
observation with schema, question, and hints.
|
| 101 |
+
"""
|
| 102 |
+
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 103 |
+
self._previous_queries = []
|
| 104 |
+
self._best_score = 0.0
|
| 105 |
+
self._step_rewards = []
|
| 106 |
+
|
| 107 |
+
self._current_task = self._select_task(task_id)
|
| 108 |
+
self._current_schema_id = self._current_task.schema_id
|
| 109 |
+
|
| 110 |
+
# close old connection if any
|
| 111 |
+
if self._db:
|
| 112 |
+
try:
|
| 113 |
+
self._db.close()
|
| 114 |
+
except Exception:
|
| 115 |
+
pass
|
| 116 |
+
self._db = create_database(self._current_schema_id)
|
| 117 |
+
|
| 118 |
+
schema = SCHEMAS[self._current_schema_id]
|
| 119 |
+
expected_rows = get_expected_result(self._db, self._current_task.ground_truth_query)
|
| 120 |
+
|
| 121 |
+
return SqlQueryObservation(
|
| 122 |
+
task_id=self._current_task.id,
|
| 123 |
+
difficulty=self._current_task.difficulty,
|
| 124 |
+
database_domain=schema.name,
|
| 125 |
+
question=self._current_task.question,
|
| 126 |
+
schema_description=schema.description,
|
| 127 |
+
query_result="",
|
| 128 |
+
query_error=None,
|
| 129 |
+
feedback=(
|
| 130 |
+
f"Task: {self._current_task.description}\n"
|
| 131 |
+
f"Database: {schema.name}\n"
|
| 132 |
+
f"Difficulty: {self._current_task.difficulty}\n"
|
| 133 |
+
f"Attempts available: {self._current_task.max_steps}\n"
|
| 134 |
+
f"\nHints:\n" + "\n".join(f" - {h}" for h in self._current_task.hints[:1])
|
| 135 |
+
),
|
| 136 |
+
diagnostics=[],
|
| 137 |
+
efficiency_notes=[],
|
| 138 |
+
expected_row_count=len(expected_rows),
|
| 139 |
+
expected_columns=self._current_task.expected_columns,
|
| 140 |
+
steps_remaining=self._current_task.max_steps,
|
| 141 |
+
current_score=0.0,
|
| 142 |
+
history=[],
|
| 143 |
+
done=False,
|
| 144 |
+
reward=0.0,
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
def _progressive_hints(self, step: int) -> str:
|
| 148 |
+
"""Reveal more hints as the agent struggles.
|
| 149 |
+
|
| 150 |
+
Step 1: just the first hint
|
| 151 |
+
Step 2-3: a few more
|
| 152 |
+
Step 4+: everything we've got
|
| 153 |
+
"""
|
| 154 |
+
task = self._current_task
|
| 155 |
+
hints = task.hints
|
| 156 |
+
|
| 157 |
+
if step <= 1:
|
| 158 |
+
shown = hints[:1] if hints else []
|
| 159 |
+
elif step <= 3:
|
| 160 |
+
shown = hints[:min(step + 1, len(hints))]
|
| 161 |
+
else:
|
| 162 |
+
shown = hints
|
| 163 |
+
|
| 164 |
+
if not shown:
|
| 165 |
+
return ""
|
| 166 |
+
return "\n".join(f" - {h}" for h in shown)
|
| 167 |
+
|
| 168 |
+
def step(self, action: SqlQueryAction) -> SqlQueryObservation:
|
| 169 |
+
"""Evaluate a submitted SQL query and return graded feedback."""
|
| 170 |
+
if self._current_task is None or self._db is None:
|
| 171 |
+
return SqlQueryObservation(
|
| 172 |
+
feedback="Environment not initialized. Call reset() first.",
|
| 173 |
+
done=True,
|
| 174 |
+
reward=0.0,
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
self._state.step_count += 1
|
| 178 |
+
step = self._state.step_count
|
| 179 |
+
steps_remaining = self._current_task.max_steps - step
|
| 180 |
+
|
| 181 |
+
# grade it
|
| 182 |
+
grade_result = grade_query(
|
| 183 |
+
conn=self._db,
|
| 184 |
+
query=action.query,
|
| 185 |
+
task=self._current_task,
|
| 186 |
+
previous_queries=self._previous_queries,
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
self._previous_queries.append(action.query)
|
| 190 |
+
self._step_rewards.append(grade_result.total_score)
|
| 191 |
+
self._best_score = max(self._best_score, grade_result.total_score)
|
| 192 |
+
|
| 193 |
+
# done if near-perfect or out of steps
|
| 194 |
+
is_done = grade_result.total_score >= 0.95 or steps_remaining <= 0
|
| 195 |
+
|
| 196 |
+
# build history for the observation
|
| 197 |
+
history = [
|
| 198 |
+
{
|
| 199 |
+
"step": i + 1,
|
| 200 |
+
"query": q[:100] + "..." if len(q) > 100 else q,
|
| 201 |
+
"score": round(s, 2),
|
| 202 |
+
}
|
| 203 |
+
for i, (q, s) in enumerate(
|
| 204 |
+
zip(self._previous_queries, self._step_rewards)
|
| 205 |
+
)
|
| 206 |
+
]
|
| 207 |
+
|
| 208 |
+
feedback = grade_result.feedback
|
| 209 |
+
|
| 210 |
+
# add hints if the agent is struggling
|
| 211 |
+
if not is_done and grade_result.total_score < 0.95:
|
| 212 |
+
hint_text = self._progressive_hints(step)
|
| 213 |
+
if hint_text:
|
| 214 |
+
feedback += f"\n\nHints (attempt {step}):\n{hint_text}"
|
| 215 |
+
|
| 216 |
+
if is_done:
|
| 217 |
+
if grade_result.total_score >= 0.95:
|
| 218 |
+
feedback += "\n\nNear-perfect score achieved."
|
| 219 |
+
else:
|
| 220 |
+
feedback += f"\n\nEpisode finished. Best score: {self._best_score:.2f}"
|
| 221 |
+
feedback += f"\nScore progression: {' -> '.join(f'{s:.2f}' for s in self._step_rewards)}"
|
| 222 |
+
|
| 223 |
+
expected_rows = get_expected_result(self._db, self._current_task.ground_truth_query)
|
| 224 |
+
schema = SCHEMAS[self._current_schema_id]
|
| 225 |
+
|
| 226 |
+
return SqlQueryObservation(
|
| 227 |
+
task_id=self._current_task.id,
|
| 228 |
+
difficulty=self._current_task.difficulty,
|
| 229 |
+
database_domain=schema.name,
|
| 230 |
+
question=self._current_task.question,
|
| 231 |
+
schema_description=schema.description,
|
| 232 |
+
query_result=grade_result.query_result or "",
|
| 233 |
+
query_error=grade_result.error,
|
| 234 |
+
feedback=feedback,
|
| 235 |
+
diagnostics=[d.to_dict() for d in grade_result.diagnostics],
|
| 236 |
+
efficiency_notes=grade_result.efficiency_notes,
|
| 237 |
+
expected_row_count=len(expected_rows),
|
| 238 |
+
expected_columns=self._current_task.expected_columns,
|
| 239 |
+
steps_remaining=max(0, steps_remaining),
|
| 240 |
+
current_score=self._best_score,
|
| 241 |
+
history=history,
|
| 242 |
+
done=is_done,
|
| 243 |
+
reward=grade_result.total_score,
|
| 244 |
+
metadata={
|
| 245 |
+
"task_id": self._current_task.id,
|
| 246 |
+
"difficulty": self._current_task.difficulty,
|
| 247 |
+
"database_domain": self._current_schema_id,
|
| 248 |
+
"step": step,
|
| 249 |
+
"best_score": self._best_score,
|
| 250 |
+
"syntax_score": grade_result.syntax_score,
|
| 251 |
+
"table_score": grade_result.table_score,
|
| 252 |
+
"column_score": grade_result.column_score,
|
| 253 |
+
"result_score": grade_result.result_score,
|
| 254 |
+
"efficiency_score": grade_result.efficiency_score,
|
| 255 |
+
"penalty": grade_result.penalty,
|
| 256 |
+
"challenge_type": self._current_task.challenge_type,
|
| 257 |
+
},
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
@property
|
| 261 |
+
def state(self) -> State:
|
| 262 |
+
return self._state
|
| 263 |
+
|
| 264 |
+
def close(self):
|
| 265 |
+
if self._db:
|
| 266 |
+
try:
|
| 267 |
+
self._db.close()
|
| 268 |
+
except Exception:
|
| 269 |
+
pass
|
| 270 |
+
self._db = None
|
server/tasks.py
ADDED
|
@@ -0,0 +1,1068 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Database schemas, seed data, and task definitions.
|
| 3 |
+
|
| 4 |
+
Three separate database domains so the agent can't just memorize one
|
| 5 |
+
schema. We experimented with more domains but three felt like the sweet
|
| 6 |
+
spot for coverage without making the task set unmanageable.
|
| 7 |
+
|
| 8 |
+
Seed data is intentionally small (~10-15 rows per table). Bigger datasets
|
| 9 |
+
would be more realistic but for an RL environment the grading speed matters
|
| 10 |
+
more. We can always scale up the data later if needed.
|
| 11 |
+
|
| 12 |
+
NOTE: if you add a new task, make sure to also add it to the openenv.yaml
|
| 13 |
+
manifest and update the TASK_IDS list in inference.py.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import sqlite3
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
from typing import Dict, List, Optional, Tuple
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
# ---------------------------------------------------------------------------
|
| 22 |
+
# Data structures
|
| 23 |
+
# ---------------------------------------------------------------------------
|
| 24 |
+
|
| 25 |
+
@dataclass
|
| 26 |
+
class DatabaseSchema:
|
| 27 |
+
"""A database domain with its own schema, seed data, and description."""
|
| 28 |
+
id: str
|
| 29 |
+
name: str
|
| 30 |
+
schema_sql: str
|
| 31 |
+
seed_sql: str
|
| 32 |
+
description: str
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass
|
| 36 |
+
class Task:
|
| 37 |
+
"""A single SQL query task with ground truth."""
|
| 38 |
+
id: str
|
| 39 |
+
difficulty: str # "easy", "medium", "hard"
|
| 40 |
+
schema_id: str # Which database schema to use
|
| 41 |
+
question: str
|
| 42 |
+
ground_truth_query: str
|
| 43 |
+
expected_columns: List[str]
|
| 44 |
+
description: str = ""
|
| 45 |
+
hints: List[str] = field(default_factory=list)
|
| 46 |
+
max_steps: int = 5
|
| 47 |
+
challenge_type: str = "" # e.g., "ambiguous", "temporal", "efficiency"
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
# ---------------------------------------------------------------------------
|
| 51 |
+
# Schema 1: Company Analytics
|
| 52 |
+
# ---------------------------------------------------------------------------
|
| 53 |
+
|
| 54 |
+
COMPANY_SCHEMA_SQL = """
|
| 55 |
+
CREATE TABLE departments (
|
| 56 |
+
id INTEGER PRIMARY KEY,
|
| 57 |
+
name TEXT NOT NULL,
|
| 58 |
+
budget REAL NOT NULL,
|
| 59 |
+
location TEXT NOT NULL
|
| 60 |
+
);
|
| 61 |
+
|
| 62 |
+
CREATE TABLE employees (
|
| 63 |
+
id INTEGER PRIMARY KEY,
|
| 64 |
+
name TEXT NOT NULL,
|
| 65 |
+
email TEXT NOT NULL,
|
| 66 |
+
department_id INTEGER NOT NULL,
|
| 67 |
+
salary REAL NOT NULL,
|
| 68 |
+
hire_date TEXT NOT NULL,
|
| 69 |
+
manager_id INTEGER,
|
| 70 |
+
is_active INTEGER DEFAULT 1,
|
| 71 |
+
FOREIGN KEY (department_id) REFERENCES departments(id),
|
| 72 |
+
FOREIGN KEY (manager_id) REFERENCES employees(id)
|
| 73 |
+
);
|
| 74 |
+
|
| 75 |
+
CREATE TABLE products (
|
| 76 |
+
id INTEGER PRIMARY KEY,
|
| 77 |
+
name TEXT NOT NULL,
|
| 78 |
+
category TEXT NOT NULL,
|
| 79 |
+
price REAL NOT NULL,
|
| 80 |
+
stock INTEGER NOT NULL DEFAULT 0,
|
| 81 |
+
created_date TEXT NOT NULL
|
| 82 |
+
);
|
| 83 |
+
|
| 84 |
+
CREATE TABLE customers (
|
| 85 |
+
id INTEGER PRIMARY KEY,
|
| 86 |
+
name TEXT NOT NULL,
|
| 87 |
+
email TEXT NOT NULL,
|
| 88 |
+
city TEXT NOT NULL,
|
| 89 |
+
country TEXT NOT NULL,
|
| 90 |
+
join_date TEXT NOT NULL,
|
| 91 |
+
tier TEXT NOT NULL DEFAULT 'standard'
|
| 92 |
+
);
|
| 93 |
+
|
| 94 |
+
CREATE TABLE orders (
|
| 95 |
+
id INTEGER PRIMARY KEY,
|
| 96 |
+
customer_id INTEGER NOT NULL,
|
| 97 |
+
product_id INTEGER NOT NULL,
|
| 98 |
+
quantity INTEGER NOT NULL,
|
| 99 |
+
total_price REAL NOT NULL,
|
| 100 |
+
order_date TEXT NOT NULL,
|
| 101 |
+
status TEXT NOT NULL DEFAULT 'pending',
|
| 102 |
+
FOREIGN KEY (customer_id) REFERENCES customers(id),
|
| 103 |
+
FOREIGN KEY (product_id) REFERENCES products(id)
|
| 104 |
+
);
|
| 105 |
+
|
| 106 |
+
CREATE TABLE reviews (
|
| 107 |
+
id INTEGER PRIMARY KEY,
|
| 108 |
+
product_id INTEGER NOT NULL,
|
| 109 |
+
customer_id INTEGER NOT NULL,
|
| 110 |
+
rating INTEGER NOT NULL CHECK (rating BETWEEN 1 AND 5),
|
| 111 |
+
review_text TEXT,
|
| 112 |
+
review_date TEXT NOT NULL,
|
| 113 |
+
FOREIGN KEY (product_id) REFERENCES products(id),
|
| 114 |
+
FOREIGN KEY (customer_id) REFERENCES customers(id)
|
| 115 |
+
);
|
| 116 |
+
"""
|
| 117 |
+
|
| 118 |
+
COMPANY_SEED_SQL = """
|
| 119 |
+
INSERT INTO departments VALUES (1, 'Engineering', 500000.00, 'San Francisco');
|
| 120 |
+
INSERT INTO departments VALUES (2, 'Marketing', 250000.00, 'New York');
|
| 121 |
+
INSERT INTO departments VALUES (3, 'Sales', 300000.00, 'Chicago');
|
| 122 |
+
INSERT INTO departments VALUES (4, 'Human Resources', 150000.00, 'San Francisco');
|
| 123 |
+
INSERT INTO departments VALUES (5, 'Finance', 200000.00, 'New York');
|
| 124 |
+
|
| 125 |
+
INSERT INTO employees VALUES (1, 'Alice Chen', 'alice@company.com', 1, 145000, '2020-03-15', NULL, 1);
|
| 126 |
+
INSERT INTO employees VALUES (2, 'Bob Smith', 'bob@company.com', 1, 130000, '2021-06-01', 1, 1);
|
| 127 |
+
INSERT INTO employees VALUES (3, 'Carol Davis', 'carol@company.com', 2, 95000, '2019-11-20', NULL, 1);
|
| 128 |
+
INSERT INTO employees VALUES (4, 'David Wilson', 'david@company.com', 1, 125000, '2022-01-10', 1, 1);
|
| 129 |
+
INSERT INTO employees VALUES (5, 'Eva Martinez', 'eva@company.com', 3, 110000, '2020-08-05', NULL, 1);
|
| 130 |
+
INSERT INTO employees VALUES (6, 'Frank Brown', 'frank@company.com', 3, 98000, '2021-04-22', 5, 1);
|
| 131 |
+
INSERT INTO employees VALUES (7, 'Grace Lee', 'grace@company.com', 2, 105000, '2022-07-14', 3, 1);
|
| 132 |
+
INSERT INTO employees VALUES (8, 'Henry Taylor', 'henry@company.com', 4, 88000, '2023-02-28', NULL, 1);
|
| 133 |
+
INSERT INTO employees VALUES (9, 'Ivy Nguyen', 'ivy@company.com', 5, 115000, '2020-12-01', NULL, 1);
|
| 134 |
+
INSERT INTO employees VALUES (10, 'Jack Anderson', 'jack@company.com', 1, 140000, '2019-09-10', 1, 1);
|
| 135 |
+
INSERT INTO employees VALUES (11, 'Karen White', 'karen@company.com', 5, 105000, '2021-10-15', 9, 1);
|
| 136 |
+
INSERT INTO employees VALUES (12, 'Leo Garcia', 'leo@company.com', 3, 92000, '2023-05-20', 5, 0);
|
| 137 |
+
INSERT INTO employees VALUES (13, 'Mia Johnson', 'mia@company.com', 2, 88000, '2022-11-08', 3, 1);
|
| 138 |
+
INSERT INTO employees VALUES (14, 'Nathan Park', 'nathan@company.com', 1, 155000, '2018-06-25', NULL, 1);
|
| 139 |
+
INSERT INTO employees VALUES (15, 'Olivia Kim', 'olivia@company.com', 4, 82000, '2023-08-12', 8, 1);
|
| 140 |
+
|
| 141 |
+
INSERT INTO products VALUES (1, 'Laptop Pro 15', 'Electronics', 1299.99, 45, '2023-01-15');
|
| 142 |
+
INSERT INTO products VALUES (2, 'Wireless Mouse', 'Electronics', 29.99, 200, '2023-02-20');
|
| 143 |
+
INSERT INTO products VALUES (3, 'Standing Desk', 'Furniture', 549.99, 30, '2023-03-10');
|
| 144 |
+
INSERT INTO products VALUES (4, 'Noise-Canceling Headphones', 'Electronics', 199.99, 75, '2023-01-25');
|
| 145 |
+
INSERT INTO products VALUES (5, 'Ergonomic Chair', 'Furniture', 399.99, 25, '2023-04-05');
|
| 146 |
+
INSERT INTO products VALUES (6, 'USB-C Hub', 'Electronics', 49.99, 150, '2023-05-12');
|
| 147 |
+
INSERT INTO products VALUES (7, 'Monitor 27inch', 'Electronics', 449.99, 60, '2023-02-14');
|
| 148 |
+
INSERT INTO products VALUES (8, 'Keyboard Mechanical', 'Electronics', 89.99, 100, '2023-06-01');
|
| 149 |
+
INSERT INTO products VALUES (9, 'Desk Lamp', 'Furniture', 39.99, 80, '2023-03-22');
|
| 150 |
+
INSERT INTO products VALUES (10, 'Webcam HD', 'Electronics', 79.99, 90, '2023-07-08');
|
| 151 |
+
|
| 152 |
+
INSERT INTO customers VALUES (1, 'TechCorp Inc', 'orders@techcorp.com', 'San Francisco', 'USA', '2023-01-10', 'premium');
|
| 153 |
+
INSERT INTO customers VALUES (2, 'DataFlow Ltd', 'buy@dataflow.io', 'London', 'UK', '2023-02-15', 'standard');
|
| 154 |
+
INSERT INTO customers VALUES (3, 'StartupXYZ', 'procurement@startupxyz.com', 'Berlin', 'Germany', '2023-03-20', 'standard');
|
| 155 |
+
INSERT INTO customers VALUES (4, 'MegaCorp', 'orders@megacorp.com', 'Tokyo', 'Japan', '2023-01-05', 'premium');
|
| 156 |
+
INSERT INTO customers VALUES (5, 'CloudNine', 'buy@cloudnine.com', 'New York', 'USA', '2023-04-12', 'premium');
|
| 157 |
+
INSERT INTO customers VALUES (6, 'InnovateLab', 'purchasing@innovatelab.com', 'Mumbai', 'India', '2023-05-08', 'standard');
|
| 158 |
+
INSERT INTO customers VALUES (7, 'AlphaWorks', 'orders@alphaworks.net', 'Sydney', 'Australia', '2023-02-28', 'standard');
|
| 159 |
+
INSERT INTO customers VALUES (8, 'BetaSoft', 'buy@betasoft.co', 'Toronto', 'Canada', '2023-06-15', 'premium');
|
| 160 |
+
|
| 161 |
+
INSERT INTO orders VALUES (1, 1, 1, 5, 6499.95, '2024-01-15', 'completed');
|
| 162 |
+
INSERT INTO orders VALUES (2, 1, 4, 10, 1999.90, '2024-01-20', 'completed');
|
| 163 |
+
INSERT INTO orders VALUES (3, 2, 2, 20, 599.80, '2024-02-05', 'completed');
|
| 164 |
+
INSERT INTO orders VALUES (4, 3, 3, 3, 1649.97, '2024-02-10', 'completed');
|
| 165 |
+
INSERT INTO orders VALUES (5, 4, 1, 8, 10399.92, '2024-02-15', 'completed');
|
| 166 |
+
INSERT INTO orders VALUES (6, 5, 7, 4, 1799.96, '2024-03-01', 'completed');
|
| 167 |
+
INSERT INTO orders VALUES (7, 1, 6, 15, 749.85, '2024-03-10', 'completed');
|
| 168 |
+
INSERT INTO orders VALUES (8, 6, 5, 2, 799.98, '2024-03-15', 'shipped');
|
| 169 |
+
INSERT INTO orders VALUES (9, 2, 8, 5, 449.95, '2024-03-20', 'shipped');
|
| 170 |
+
INSERT INTO orders VALUES (10, 4, 2, 50, 1499.50, '2024-04-01', 'completed');
|
| 171 |
+
INSERT INTO orders VALUES (11, 7, 9, 10, 399.90, '2024-04-05', 'completed');
|
| 172 |
+
INSERT INTO orders VALUES (12, 3, 10, 6, 479.94, '2024-04-10', 'shipped');
|
| 173 |
+
INSERT INTO orders VALUES (13, 8, 1, 3, 3899.97, '2024-04-15', 'pending');
|
| 174 |
+
INSERT INTO orders VALUES (14, 5, 5, 5, 1999.95, '2024-04-20', 'completed');
|
| 175 |
+
INSERT INTO orders VALUES (15, 6, 8, 8, 719.92, '2024-05-01', 'completed');
|
| 176 |
+
INSERT INTO orders VALUES (16, 1, 3, 2, 1099.98, '2024-05-05', 'completed');
|
| 177 |
+
INSERT INTO orders VALUES (17, 4, 6, 25, 1249.75, '2024-05-10', 'shipped');
|
| 178 |
+
INSERT INTO orders VALUES (18, 7, 4, 3, 599.97, '2024-05-15', 'completed');
|
| 179 |
+
INSERT INTO orders VALUES (19, 2, 7, 2, 899.98, '2024-05-20', 'pending');
|
| 180 |
+
INSERT INTO orders VALUES (20, 8, 10, 4, 319.96, '2024-05-25', 'completed');
|
| 181 |
+
INSERT INTO orders VALUES (21, 5, 2, 30, 899.70, '2024-06-01', 'completed');
|
| 182 |
+
INSERT INTO orders VALUES (22, 3, 1, 2, 2599.98, '2024-06-05', 'completed');
|
| 183 |
+
INSERT INTO orders VALUES (23, 6, 3, 1, 549.99, '2024-06-10', 'shipped');
|
| 184 |
+
INSERT INTO orders VALUES (24, 4, 8, 12, 1079.88, '2024-06-15', 'completed');
|
| 185 |
+
|
| 186 |
+
INSERT INTO reviews VALUES (1, 1, 1, 5, 'Excellent laptop, very fast!', '2024-02-01');
|
| 187 |
+
INSERT INTO reviews VALUES (2, 1, 4, 4, 'Good performance but heavy', '2024-03-01');
|
| 188 |
+
INSERT INTO reviews VALUES (3, 2, 2, 4, 'Good mouse, comfortable grip', '2024-02-20');
|
| 189 |
+
INSERT INTO reviews VALUES (4, 3, 3, 5, 'Best standing desk ever!', '2024-03-01');
|
| 190 |
+
INSERT INTO reviews VALUES (5, 4, 1, 5, 'Amazing noise cancellation', '2024-02-10');
|
| 191 |
+
INSERT INTO reviews VALUES (6, 5, 6, 3, 'Decent chair but could be better', '2024-04-01');
|
| 192 |
+
INSERT INTO reviews VALUES (7, 7, 5, 4, 'Great display quality', '2024-03-15');
|
| 193 |
+
INSERT INTO reviews VALUES (8, 8, 2, 5, 'Perfect mechanical keyboard', '2024-04-05');
|
| 194 |
+
INSERT INTO reviews VALUES (9, 6, 1, 4, 'Works well with all my devices', '2024-03-20');
|
| 195 |
+
INSERT INTO reviews VALUES (10, 10, 7, 3, 'Average webcam quality', '2024-05-01');
|
| 196 |
+
INSERT INTO reviews VALUES (11, 1, 8, 5, 'Worth every penny', '2024-05-10');
|
| 197 |
+
INSERT INTO reviews VALUES (12, 4, 5, 4, 'Great sound quality', '2024-05-20');
|
| 198 |
+
INSERT INTO reviews VALUES (13, 9, 3, 4, 'Nice desk lamp, adjustable', '2024-04-15');
|
| 199 |
+
INSERT INTO reviews VALUES (14, 5, 8, 2, 'Armrests broke after 2 months', '2024-06-01');
|
| 200 |
+
INSERT INTO reviews VALUES (15, 2, 6, 4, 'Reliable and affordable', '2024-06-10');
|
| 201 |
+
"""
|
| 202 |
+
|
| 203 |
+
COMPANY_DESCRIPTION = """DATABASE SCHEMA β Company Analytics
|
| 204 |
+
=======================================
|
| 205 |
+
|
| 206 |
+
TABLE: departments
|
| 207 |
+
- id (INTEGER, PK) | name (TEXT) | budget (REAL) | location (TEXT)
|
| 208 |
+
|
| 209 |
+
TABLE: employees
|
| 210 |
+
- id (INTEGER, PK) | name (TEXT) | email (TEXT) | department_id (FKβdepartments)
|
| 211 |
+
- salary (REAL) | hire_date (TEXT, YYYY-MM-DD) | manager_id (FKβemployees, nullable)
|
| 212 |
+
- is_active (INTEGER, 1=active/0=inactive)
|
| 213 |
+
|
| 214 |
+
TABLE: products
|
| 215 |
+
- id (INTEGER, PK) | name (TEXT) | category (TEXT, 'Electronics'/'Furniture')
|
| 216 |
+
- price (REAL) | stock (INTEGER) | created_date (TEXT)
|
| 217 |
+
|
| 218 |
+
TABLE: customers
|
| 219 |
+
- id (INTEGER, PK) | name (TEXT) | email (TEXT) | city (TEXT) | country (TEXT)
|
| 220 |
+
- join_date (TEXT) | tier (TEXT, 'standard'/'premium')
|
| 221 |
+
|
| 222 |
+
TABLE: orders
|
| 223 |
+
- id (INTEGER, PK) | customer_id (FKβcustomers) | product_id (FKβproducts)
|
| 224 |
+
- quantity (INTEGER) | total_price (REAL) | order_date (TEXT)
|
| 225 |
+
- status (TEXT, 'pending'/'shipped'/'completed')
|
| 226 |
+
|
| 227 |
+
TABLE: reviews
|
| 228 |
+
- id (INTEGER, PK) | product_id (FKβproducts) | customer_id (FKβcustomers)
|
| 229 |
+
- rating (INTEGER, 1-5) | review_text (TEXT, nullable) | review_date (TEXT)
|
| 230 |
+
|
| 231 |
+
RELATIONSHIPS:
|
| 232 |
+
employees.department_id β departments.id
|
| 233 |
+
employees.manager_id β employees.id (self-referencing, hierarchical)
|
| 234 |
+
orders.customer_id β customers.id
|
| 235 |
+
orders.product_id β products.id
|
| 236 |
+
reviews.product_id β products.id
|
| 237 |
+
reviews.customer_id β customers.id
|
| 238 |
+
"""
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
# ---------------------------------------------------------------------------
|
| 242 |
+
# Schema 2: Hospital Management
|
| 243 |
+
# ---------------------------------------------------------------------------
|
| 244 |
+
|
| 245 |
+
HOSPITAL_SCHEMA_SQL = """
|
| 246 |
+
CREATE TABLE wards (
|
| 247 |
+
id INTEGER PRIMARY KEY,
|
| 248 |
+
name TEXT NOT NULL,
|
| 249 |
+
capacity INTEGER NOT NULL,
|
| 250 |
+
floor INTEGER NOT NULL
|
| 251 |
+
);
|
| 252 |
+
|
| 253 |
+
CREATE TABLE doctors (
|
| 254 |
+
id INTEGER PRIMARY KEY,
|
| 255 |
+
name TEXT NOT NULL,
|
| 256 |
+
specialization TEXT NOT NULL,
|
| 257 |
+
ward_id INTEGER NOT NULL,
|
| 258 |
+
experience_years INTEGER NOT NULL,
|
| 259 |
+
is_available INTEGER DEFAULT 1,
|
| 260 |
+
FOREIGN KEY (ward_id) REFERENCES wards(id)
|
| 261 |
+
);
|
| 262 |
+
|
| 263 |
+
CREATE TABLE patients (
|
| 264 |
+
id INTEGER PRIMARY KEY,
|
| 265 |
+
name TEXT NOT NULL,
|
| 266 |
+
age INTEGER NOT NULL,
|
| 267 |
+
gender TEXT NOT NULL,
|
| 268 |
+
ward_id INTEGER,
|
| 269 |
+
admission_date TEXT NOT NULL,
|
| 270 |
+
discharge_date TEXT,
|
| 271 |
+
diagnosis TEXT NOT NULL,
|
| 272 |
+
insurance_type TEXT NOT NULL DEFAULT 'none',
|
| 273 |
+
FOREIGN KEY (ward_id) REFERENCES wards(id)
|
| 274 |
+
);
|
| 275 |
+
|
| 276 |
+
CREATE TABLE appointments (
|
| 277 |
+
id INTEGER PRIMARY KEY,
|
| 278 |
+
patient_id INTEGER NOT NULL,
|
| 279 |
+
doctor_id INTEGER NOT NULL,
|
| 280 |
+
appointment_date TEXT NOT NULL,
|
| 281 |
+
appointment_type TEXT NOT NULL,
|
| 282 |
+
status TEXT NOT NULL DEFAULT 'scheduled',
|
| 283 |
+
notes TEXT,
|
| 284 |
+
FOREIGN KEY (patient_id) REFERENCES patients(id),
|
| 285 |
+
FOREIGN KEY (doctor_id) REFERENCES doctors(id)
|
| 286 |
+
);
|
| 287 |
+
|
| 288 |
+
CREATE TABLE medications (
|
| 289 |
+
id INTEGER PRIMARY KEY,
|
| 290 |
+
name TEXT NOT NULL,
|
| 291 |
+
category TEXT NOT NULL,
|
| 292 |
+
unit_cost REAL NOT NULL
|
| 293 |
+
);
|
| 294 |
+
|
| 295 |
+
CREATE TABLE prescriptions (
|
| 296 |
+
id INTEGER PRIMARY KEY,
|
| 297 |
+
patient_id INTEGER NOT NULL,
|
| 298 |
+
doctor_id INTEGER NOT NULL,
|
| 299 |
+
medication_id INTEGER NOT NULL,
|
| 300 |
+
dosage TEXT NOT NULL,
|
| 301 |
+
frequency TEXT NOT NULL,
|
| 302 |
+
start_date TEXT NOT NULL,
|
| 303 |
+
end_date TEXT,
|
| 304 |
+
FOREIGN KEY (patient_id) REFERENCES patients(id),
|
| 305 |
+
FOREIGN KEY (doctor_id) REFERENCES doctors(id),
|
| 306 |
+
FOREIGN KEY (medication_id) REFERENCES medications(id)
|
| 307 |
+
);
|
| 308 |
+
"""
|
| 309 |
+
|
| 310 |
+
HOSPITAL_SEED_SQL = """
|
| 311 |
+
INSERT INTO wards VALUES (1, 'Cardiology', 30, 2);
|
| 312 |
+
INSERT INTO wards VALUES (2, 'Neurology', 20, 3);
|
| 313 |
+
INSERT INTO wards VALUES (3, 'Orthopedics', 25, 1);
|
| 314 |
+
INSERT INTO wards VALUES (4, 'Pediatrics', 35, 1);
|
| 315 |
+
INSERT INTO wards VALUES (5, 'Emergency', 40, 0);
|
| 316 |
+
|
| 317 |
+
INSERT INTO doctors VALUES (1, 'Dr. Sarah Patel', 'Cardiologist', 1, 15, 1);
|
| 318 |
+
INSERT INTO doctors VALUES (2, 'Dr. James Liu', 'Neurologist', 2, 12, 1);
|
| 319 |
+
INSERT INTO doctors VALUES (3, 'Dr. Maria Garcia', 'Orthopedic Surgeon', 3, 20, 1);
|
| 320 |
+
INSERT INTO doctors VALUES (4, 'Dr. Robert Kim', 'Pediatrician', 4, 8, 1);
|
| 321 |
+
INSERT INTO doctors VALUES (5, 'Dr. Emily Chen', 'Emergency Medicine', 5, 10, 1);
|
| 322 |
+
INSERT INTO doctors VALUES (6, 'Dr. Michael Brown', 'Cardiologist', 1, 5, 0);
|
| 323 |
+
INSERT INTO doctors VALUES (7, 'Dr. Lisa Wong', 'Neurologist', 2, 18, 1);
|
| 324 |
+
INSERT INTO doctors VALUES (8, 'Dr. David Okafor', 'Pediatrician', 4, 3, 1);
|
| 325 |
+
|
| 326 |
+
INSERT INTO patients VALUES (1, 'John Doe', 65, 'M', 1, '2024-01-10', '2024-01-20', 'Chest Pain', 'private');
|
| 327 |
+
INSERT INTO patients VALUES (2, 'Jane Smith', 45, 'F', 2, '2024-01-15', '2024-02-01', 'Migraine', 'government');
|
| 328 |
+
INSERT INTO patients VALUES (3, 'Bob Johnson', 72, 'M', 1, '2024-02-01', NULL, 'Heart Failure', 'private');
|
| 329 |
+
INSERT INTO patients VALUES (4, 'Alice Williams', 8, 'F', 4, '2024-02-10', '2024-02-15', 'Flu', 'private');
|
| 330 |
+
INSERT INTO patients VALUES (5, 'Charlie Brown', 55, 'M', 3, '2024-02-20', '2024-03-15', 'Fracture', 'government');
|
| 331 |
+
INSERT INTO patients VALUES (6, 'Diana Ross', 38, 'F', 5, '2024-03-01', '2024-03-02', 'Laceration', 'none');
|
| 332 |
+
INSERT INTO patients VALUES (7, 'Edward Lee', 80, 'M', 1, '2024-03-10', NULL, 'Arrhythmia', 'government');
|
| 333 |
+
INSERT INTO patients VALUES (8, 'Fiona Clark', 12, 'F', 4, '2024-03-15', '2024-03-18', 'Asthma', 'private');
|
| 334 |
+
INSERT INTO patients VALUES (9, 'George Martin', 60, 'M', 2, '2024-03-20', '2024-04-05', 'Stroke', 'private');
|
| 335 |
+
INSERT INTO patients VALUES (10, 'Helen Park', 50, 'F', 3, '2024-04-01', NULL, 'Joint Replacement', 'government');
|
| 336 |
+
INSERT INTO patients VALUES (11, 'Ivan Petrov', 70, 'M', 5, '2024-04-10', '2024-04-11', 'Chest Pain', 'none');
|
| 337 |
+
INSERT INTO patients VALUES (12, 'Julia Santos', 5, 'F', 4, '2024-04-15', '2024-04-17', 'Ear Infection', 'private');
|
| 338 |
+
|
| 339 |
+
INSERT INTO appointments VALUES (1, 1, 1, '2024-01-12', 'Follow-up', 'completed', 'ECG normal');
|
| 340 |
+
INSERT INTO appointments VALUES (2, 2, 2, '2024-01-20', 'Consultation', 'completed', 'MRI scheduled');
|
| 341 |
+
INSERT INTO appointments VALUES (3, 3, 1, '2024-02-05', 'Initial', 'completed', 'Admitted to ICU');
|
| 342 |
+
INSERT INTO appointments VALUES (4, 4, 4, '2024-02-12', 'Follow-up', 'completed', 'Recovering well');
|
| 343 |
+
INSERT INTO appointments VALUES (5, 5, 3, '2024-02-25', 'Surgery', 'completed', 'Pin insertion');
|
| 344 |
+
INSERT INTO appointments VALUES (6, 6, 5, '2024-03-01', 'Emergency', 'completed', 'Stitches applied');
|
| 345 |
+
INSERT INTO appointments VALUES (7, 7, 1, '2024-03-12', 'Initial', 'completed', 'Pacemaker evaluation');
|
| 346 |
+
INSERT INTO appointments VALUES (8, 9, 7, '2024-03-22', 'Initial', 'completed', 'CT scan ordered');
|
| 347 |
+
INSERT INTO appointments VALUES (9, 3, 6, '2024-03-15', 'Follow-up', 'cancelled', NULL);
|
| 348 |
+
INSERT INTO appointments VALUES (10, 10, 3, '2024-04-03', 'Surgery', 'scheduled', 'Pre-op clearance needed');
|
| 349 |
+
INSERT INTO appointments VALUES (11, 11, 5, '2024-04-10', 'Emergency', 'completed', 'Ruled out MI');
|
| 350 |
+
INSERT INTO appointments VALUES (12, 1, 1, '2024-04-15', 'Follow-up', 'scheduled', NULL);
|
| 351 |
+
INSERT INTO appointments VALUES (13, 8, 4, '2024-03-16', 'Follow-up', 'completed', 'Inhaler prescribed');
|
| 352 |
+
INSERT INTO appointments VALUES (14, 9, 2, '2024-04-01', 'Follow-up', 'completed', 'Rehab started');
|
| 353 |
+
INSERT INTO appointments VALUES (15, 1, 1, '2024-05-10', 'Follow-up', 'completed', 'Stable condition');
|
| 354 |
+
INSERT INTO appointments VALUES (16, 3, 1, '2024-04-20', 'Follow-up', 'completed', 'Medication adjusted');
|
| 355 |
+
INSERT INTO appointments VALUES (17, 9, 7, '2024-04-25', 'Follow-up', 'completed', 'Good progress');
|
| 356 |
+
|
| 357 |
+
INSERT INTO medications VALUES (1, 'Aspirin', 'Blood Thinner', 0.50);
|
| 358 |
+
INSERT INTO medications VALUES (2, 'Metoprolol', 'Beta Blocker', 1.20);
|
| 359 |
+
INSERT INTO medications VALUES (3, 'Sumatriptan', 'Migraine Relief', 8.50);
|
| 360 |
+
INSERT INTO medications VALUES (4, 'Amoxicillin', 'Antibiotic', 0.80);
|
| 361 |
+
INSERT INTO medications VALUES (5, 'Ibuprofen', 'Pain Relief', 0.30);
|
| 362 |
+
INSERT INTO medications VALUES (6, 'Albuterol', 'Bronchodilator', 15.00);
|
| 363 |
+
INSERT INTO medications VALUES (7, 'Warfarin', 'Anticoagulant', 2.00);
|
| 364 |
+
INSERT INTO medications VALUES (8, 'Lisinopril', 'ACE Inhibitor', 0.90);
|
| 365 |
+
|
| 366 |
+
INSERT INTO prescriptions VALUES (1, 1, 1, 1, '81mg', 'daily', '2024-01-12', NULL);
|
| 367 |
+
INSERT INTO prescriptions VALUES (2, 1, 1, 2, '50mg', 'twice daily', '2024-01-12', '2024-03-12');
|
| 368 |
+
INSERT INTO prescriptions VALUES (3, 2, 2, 3, '50mg', 'as needed', '2024-01-20', NULL);
|
| 369 |
+
INSERT INTO prescriptions VALUES (4, 3, 1, 7, '5mg', 'daily', '2024-02-05', NULL);
|
| 370 |
+
INSERT INTO prescriptions VALUES (5, 3, 1, 8, '10mg', 'daily', '2024-02-05', NULL);
|
| 371 |
+
INSERT INTO prescriptions VALUES (6, 5, 3, 5, '400mg', 'three times daily', '2024-02-25', '2024-03-25');
|
| 372 |
+
INSERT INTO prescriptions VALUES (7, 8, 4, 6, '2 puffs', 'as needed', '2024-03-16', NULL);
|
| 373 |
+
INSERT INTO prescriptions VALUES (8, 7, 1, 2, '25mg', 'daily', '2024-03-12', NULL);
|
| 374 |
+
INSERT INTO prescriptions VALUES (9, 9, 7, 1, '325mg', 'daily', '2024-03-22', NULL);
|
| 375 |
+
INSERT INTO prescriptions VALUES (10, 12, 4, 4, '250mg', 'three times daily', '2024-04-15', '2024-04-25');
|
| 376 |
+
INSERT INTO prescriptions VALUES (11, 11, 1, 1, '81mg', 'daily', '2024-04-10', NULL);
|
| 377 |
+
INSERT INTO prescriptions VALUES (12, 8, 5, 5, '200mg', 'as needed', '2024-03-17', '2024-03-20');
|
| 378 |
+
INSERT INTO prescriptions VALUES (13, 10, 7, 5, '400mg', 'twice daily', '2024-04-05', NULL);
|
| 379 |
+
"""
|
| 380 |
+
|
| 381 |
+
HOSPITAL_DESCRIPTION = """DATABASE SCHEMA β Hospital Management
|
| 382 |
+
=======================================
|
| 383 |
+
|
| 384 |
+
TABLE: wards
|
| 385 |
+
- id (INTEGER, PK) | name (TEXT) | capacity (INTEGER) | floor (INTEGER)
|
| 386 |
+
|
| 387 |
+
TABLE: doctors
|
| 388 |
+
- id (INTEGER, PK) | name (TEXT) | specialization (TEXT)
|
| 389 |
+
- ward_id (FKβwards) | experience_years (INTEGER) | is_available (INTEGER)
|
| 390 |
+
|
| 391 |
+
TABLE: patients
|
| 392 |
+
- id (INTEGER, PK) | name (TEXT) | age (INTEGER) | gender (TEXT, 'M'/'F')
|
| 393 |
+
- ward_id (FKβwards, nullable) | admission_date (TEXT) | discharge_date (TEXT, nullable)
|
| 394 |
+
- diagnosis (TEXT) | insurance_type (TEXT, 'private'/'government'/'none')
|
| 395 |
+
|
| 396 |
+
TABLE: appointments
|
| 397 |
+
- id (INTEGER, PK) | patient_id (FKβpatients) | doctor_id (FKβdoctors)
|
| 398 |
+
- appointment_date (TEXT) | appointment_type (TEXT)
|
| 399 |
+
- status (TEXT, 'scheduled'/'completed'/'cancelled') | notes (TEXT, nullable)
|
| 400 |
+
|
| 401 |
+
TABLE: medications
|
| 402 |
+
- id (INTEGER, PK) | name (TEXT) | category (TEXT) | unit_cost (REAL)
|
| 403 |
+
|
| 404 |
+
TABLE: prescriptions
|
| 405 |
+
- id (INTEGER, PK) | patient_id (FKβpatients) | doctor_id (FKβdoctors)
|
| 406 |
+
- medication_id (FKβmedications) | dosage (TEXT) | frequency (TEXT)
|
| 407 |
+
- start_date (TEXT) | end_date (TEXT, nullable β NULL means ongoing)
|
| 408 |
+
|
| 409 |
+
RELATIONSHIPS:
|
| 410 |
+
doctors.ward_id β wards.id
|
| 411 |
+
patients.ward_id β wards.id (nullable β outpatients have no ward)
|
| 412 |
+
appointments.patient_id β patients.id
|
| 413 |
+
appointments.doctor_id β doctors.id
|
| 414 |
+
prescriptions.patient_id β patients.id
|
| 415 |
+
prescriptions.doctor_id β doctors.id
|
| 416 |
+
prescriptions.medication_id β medications.id
|
| 417 |
+
|
| 418 |
+
NOTE: discharge_date = NULL means patient is still admitted.
|
| 419 |
+
end_date = NULL in prescriptions means medication is ongoing.
|
| 420 |
+
"""
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
# ---------------------------------------------------------------------------
|
| 424 |
+
# Schema 3: E-Commerce Platform
|
| 425 |
+
# ---------------------------------------------------------------------------
|
| 426 |
+
|
| 427 |
+
ECOMMERCE_SCHEMA_SQL = """
|
| 428 |
+
CREATE TABLE sellers (
|
| 429 |
+
id INTEGER PRIMARY KEY,
|
| 430 |
+
name TEXT NOT NULL,
|
| 431 |
+
rating REAL NOT NULL,
|
| 432 |
+
city TEXT NOT NULL,
|
| 433 |
+
joined_date TEXT NOT NULL
|
| 434 |
+
);
|
| 435 |
+
|
| 436 |
+
CREATE TABLE categories (
|
| 437 |
+
id INTEGER PRIMARY KEY,
|
| 438 |
+
name TEXT NOT NULL,
|
| 439 |
+
parent_category TEXT
|
| 440 |
+
);
|
| 441 |
+
|
| 442 |
+
CREATE TABLE products (
|
| 443 |
+
id INTEGER PRIMARY KEY,
|
| 444 |
+
name TEXT NOT NULL,
|
| 445 |
+
category_id INTEGER NOT NULL,
|
| 446 |
+
seller_id INTEGER NOT NULL,
|
| 447 |
+
price REAL NOT NULL,
|
| 448 |
+
stock INTEGER NOT NULL DEFAULT 0,
|
| 449 |
+
listed_date TEXT NOT NULL,
|
| 450 |
+
FOREIGN KEY (category_id) REFERENCES categories(id),
|
| 451 |
+
FOREIGN KEY (seller_id) REFERENCES sellers(id)
|
| 452 |
+
);
|
| 453 |
+
|
| 454 |
+
CREATE TABLE users (
|
| 455 |
+
id INTEGER PRIMARY KEY,
|
| 456 |
+
name TEXT NOT NULL,
|
| 457 |
+
email TEXT NOT NULL,
|
| 458 |
+
city TEXT NOT NULL,
|
| 459 |
+
signup_date TEXT NOT NULL,
|
| 460 |
+
is_premium INTEGER DEFAULT 0
|
| 461 |
+
);
|
| 462 |
+
|
| 463 |
+
CREATE TABLE orders (
|
| 464 |
+
id INTEGER PRIMARY KEY,
|
| 465 |
+
user_id INTEGER NOT NULL,
|
| 466 |
+
order_date TEXT NOT NULL,
|
| 467 |
+
total_amount REAL NOT NULL,
|
| 468 |
+
status TEXT NOT NULL DEFAULT 'pending',
|
| 469 |
+
payment_method TEXT NOT NULL,
|
| 470 |
+
FOREIGN KEY (user_id) REFERENCES users(id)
|
| 471 |
+
);
|
| 472 |
+
|
| 473 |
+
CREATE TABLE order_items (
|
| 474 |
+
id INTEGER PRIMARY KEY,
|
| 475 |
+
order_id INTEGER NOT NULL,
|
| 476 |
+
product_id INTEGER NOT NULL,
|
| 477 |
+
quantity INTEGER NOT NULL,
|
| 478 |
+
unit_price REAL NOT NULL,
|
| 479 |
+
FOREIGN KEY (order_id) REFERENCES orders(id),
|
| 480 |
+
FOREIGN KEY (product_id) REFERENCES products(id)
|
| 481 |
+
);
|
| 482 |
+
|
| 483 |
+
CREATE TABLE returns (
|
| 484 |
+
id INTEGER PRIMARY KEY,
|
| 485 |
+
order_item_id INTEGER NOT NULL,
|
| 486 |
+
reason TEXT NOT NULL,
|
| 487 |
+
return_date TEXT NOT NULL,
|
| 488 |
+
refund_amount REAL NOT NULL,
|
| 489 |
+
status TEXT NOT NULL DEFAULT 'pending',
|
| 490 |
+
FOREIGN KEY (order_item_id) REFERENCES order_items(id)
|
| 491 |
+
);
|
| 492 |
+
"""
|
| 493 |
+
|
| 494 |
+
ECOMMERCE_SEED_SQL = """
|
| 495 |
+
INSERT INTO sellers VALUES (1, 'TechWorld', 4.5, 'Shenzhen', '2022-01-10');
|
| 496 |
+
INSERT INTO sellers VALUES (2, 'HomeStyle', 4.2, 'Mumbai', '2022-03-15');
|
| 497 |
+
INSERT INTO sellers VALUES (3, 'FashionHub', 3.8, 'Milan', '2022-06-20');
|
| 498 |
+
INSERT INTO sellers VALUES (4, 'BookBarn', 4.7, 'New York', '2021-11-01');
|
| 499 |
+
INSERT INTO sellers VALUES (5, 'SportZone', 4.0, 'Tokyo', '2023-02-14');
|
| 500 |
+
|
| 501 |
+
INSERT INTO categories VALUES (1, 'Electronics', NULL);
|
| 502 |
+
INSERT INTO categories VALUES (2, 'Smartphones', 'Electronics');
|
| 503 |
+
INSERT INTO categories VALUES (3, 'Laptops', 'Electronics');
|
| 504 |
+
INSERT INTO categories VALUES (4, 'Home & Kitchen', NULL);
|
| 505 |
+
INSERT INTO categories VALUES (5, 'Furniture', 'Home & Kitchen');
|
| 506 |
+
INSERT INTO categories VALUES (6, 'Appliances', 'Home & Kitchen');
|
| 507 |
+
INSERT INTO categories VALUES (7, 'Books', NULL);
|
| 508 |
+
INSERT INTO categories VALUES (8, 'Fiction', 'Books');
|
| 509 |
+
INSERT INTO categories VALUES (9, 'Non-Fiction', 'Books');
|
| 510 |
+
INSERT INTO categories VALUES (10, 'Sports', NULL);
|
| 511 |
+
|
| 512 |
+
INSERT INTO products VALUES (1, 'Galaxy S24', 2, 1, 899.99, 120, '2024-01-15');
|
| 513 |
+
INSERT INTO products VALUES (2, 'MacBook Air M3', 3, 1, 1199.99, 45, '2024-01-20');
|
| 514 |
+
INSERT INTO products VALUES (3, 'Blender Pro', 6, 2, 79.99, 200, '2024-02-01');
|
| 515 |
+
INSERT INTO products VALUES (4, 'Oak Dining Table', 5, 2, 599.99, 15, '2024-02-10');
|
| 516 |
+
INSERT INTO products VALUES (5, 'Running Shoes X1', 10, 5, 129.99, 300, '2024-01-05');
|
| 517 |
+
INSERT INTO products VALUES (6, 'Python Crash Course', 9, 4, 39.99, 500, '2024-03-01');
|
| 518 |
+
INSERT INTO products VALUES (7, 'The Great Gatsby', 8, 4, 12.99, 350, '2024-01-10');
|
| 519 |
+
INSERT INTO products VALUES (8, 'Yoga Mat Premium', 10, 5, 49.99, 180, '2024-02-20');
|
| 520 |
+
INSERT INTO products VALUES (9, 'Coffee Maker Deluxe', 6, 2, 149.99, 90, '2024-03-05');
|
| 521 |
+
INSERT INTO products VALUES (10, 'Pixel 8 Pro', 2, 1, 999.99, 80, '2024-03-10');
|
| 522 |
+
INSERT INTO products VALUES (11, 'Standing Desk Frame', 5, 2, 349.99, 40, '2024-03-15');
|
| 523 |
+
INSERT INTO products VALUES (12, 'Dune: Part One', 8, 4, 14.99, 200, '2024-02-01');
|
| 524 |
+
|
| 525 |
+
INSERT INTO users VALUES (1, 'Priya Sharma', 'priya@email.com', 'Mumbai', '2023-06-15', 1);
|
| 526 |
+
INSERT INTO users VALUES (2, 'Tom Baker', 'tom@email.com', 'London', '2023-08-20', 0);
|
| 527 |
+
INSERT INTO users VALUES (3, 'Yuki Tanaka', 'yuki@email.com', 'Tokyo', '2023-09-01', 1);
|
| 528 |
+
INSERT INTO users VALUES (4, 'Carlos Mendez', 'carlos@email.com', 'Mexico City', '2023-11-10', 0);
|
| 529 |
+
INSERT INTO users VALUES (5, 'Anna Kowalski', 'anna@email.com', 'Warsaw', '2024-01-05', 1);
|
| 530 |
+
INSERT INTO users VALUES (6, 'Wei Zhang', 'wei@email.com', 'Shanghai', '2024-02-14', 0);
|
| 531 |
+
|
| 532 |
+
INSERT INTO orders VALUES (1, 1, '2024-01-20', 899.99, 'delivered', 'credit_card');
|
| 533 |
+
INSERT INTO orders VALUES (2, 1, '2024-02-15', 1279.98, 'delivered', 'credit_card');
|
| 534 |
+
INSERT INTO orders VALUES (3, 2, '2024-02-01', 79.99, 'delivered', 'paypal');
|
| 535 |
+
INSERT INTO orders VALUES (4, 3, '2024-02-20', 1329.98, 'delivered', 'credit_card');
|
| 536 |
+
INSERT INTO orders VALUES (5, 4, '2024-03-01', 52.98, 'delivered', 'debit_card');
|
| 537 |
+
INSERT INTO orders VALUES (6, 5, '2024-03-10', 899.99, 'delivered', 'credit_card');
|
| 538 |
+
INSERT INTO orders VALUES (7, 1, '2024-03-15', 199.98, 'shipped', 'credit_card');
|
| 539 |
+
INSERT INTO orders VALUES (8, 2, '2024-03-20', 599.99, 'delivered', 'paypal');
|
| 540 |
+
INSERT INTO orders VALUES (9, 3, '2024-04-01', 179.98, 'delivered', 'credit_card');
|
| 541 |
+
INSERT INTO orders VALUES (10, 6, '2024-04-05', 1199.99, 'pending', 'credit_card');
|
| 542 |
+
INSERT INTO orders VALUES (11, 4, '2024-04-10', 129.99, 'delivered', 'debit_card');
|
| 543 |
+
INSERT INTO orders VALUES (12, 5, '2024-04-15', 349.99, 'cancelled', 'credit_card');
|
| 544 |
+
|
| 545 |
+
INSERT INTO order_items VALUES (1, 1, 1, 1, 899.99);
|
| 546 |
+
INSERT INTO order_items VALUES (2, 2, 2, 1, 1199.99);
|
| 547 |
+
INSERT INTO order_items VALUES (3, 2, 3, 1, 79.99);
|
| 548 |
+
INSERT INTO order_items VALUES (4, 3, 3, 1, 79.99);
|
| 549 |
+
INSERT INTO order_items VALUES (5, 4, 2, 1, 1199.99);
|
| 550 |
+
INSERT INTO order_items VALUES (6, 4, 5, 1, 129.99);
|
| 551 |
+
INSERT INTO order_items VALUES (7, 5, 6, 1, 39.99);
|
| 552 |
+
INSERT INTO order_items VALUES (8, 5, 7, 1, 12.99);
|
| 553 |
+
INSERT INTO order_items VALUES (9, 6, 1, 1, 899.99);
|
| 554 |
+
INSERT INTO order_items VALUES (10, 7, 8, 2, 49.99);
|
| 555 |
+
INSERT INTO order_items VALUES (11, 7, 9, 1, 149.99);
|
| 556 |
+
INSERT INTO order_items VALUES (12, 8, 4, 1, 599.99);
|
| 557 |
+
INSERT INTO order_items VALUES (13, 9, 5, 1, 129.99);
|
| 558 |
+
INSERT INTO order_items VALUES (14, 9, 8, 1, 49.99);
|
| 559 |
+
INSERT INTO order_items VALUES (15, 10, 2, 1, 1199.99);
|
| 560 |
+
INSERT INTO order_items VALUES (16, 11, 5, 1, 129.99);
|
| 561 |
+
INSERT INTO order_items VALUES (17, 12, 11, 1, 349.99);
|
| 562 |
+
|
| 563 |
+
INSERT INTO returns VALUES (1, 4, 'Defective product', '2024-02-10', 79.99, 'refunded');
|
| 564 |
+
INSERT INTO returns VALUES (2, 6, 'Wrong size', '2024-03-01', 129.99, 'refunded');
|
| 565 |
+
INSERT INTO returns VALUES (3, 10, 'Changed mind', '2024-03-25', 99.98, 'pending');
|
| 566 |
+
"""
|
| 567 |
+
|
| 568 |
+
ECOMMERCE_DESCRIPTION = """DATABASE SCHEMA β E-Commerce Platform
|
| 569 |
+
=======================================
|
| 570 |
+
|
| 571 |
+
TABLE: sellers
|
| 572 |
+
- id (INTEGER, PK) | name (TEXT) | rating (REAL, 0.0-5.0) | city (TEXT)
|
| 573 |
+
- joined_date (TEXT)
|
| 574 |
+
|
| 575 |
+
TABLE: categories
|
| 576 |
+
- id (INTEGER, PK) | name (TEXT) | parent_category (TEXT, nullable β NULL = top-level)
|
| 577 |
+
|
| 578 |
+
TABLE: products
|
| 579 |
+
- id (INTEGER, PK) | name (TEXT) | category_id (FKβcategories) | seller_id (FKβsellers)
|
| 580 |
+
- price (REAL) | stock (INTEGER) | listed_date (TEXT)
|
| 581 |
+
|
| 582 |
+
TABLE: users
|
| 583 |
+
- id (INTEGER, PK) | name (TEXT) | email (TEXT) | city (TEXT)
|
| 584 |
+
- signup_date (TEXT) | is_premium (INTEGER, 0/1)
|
| 585 |
+
|
| 586 |
+
TABLE: orders
|
| 587 |
+
- id (INTEGER, PK) | user_id (FKβusers) | order_date (TEXT)
|
| 588 |
+
- total_amount (REAL) | status (TEXT, 'pending'/'shipped'/'delivered'/'cancelled')
|
| 589 |
+
- payment_method (TEXT, 'credit_card'/'debit_card'/'paypal')
|
| 590 |
+
|
| 591 |
+
TABLE: order_items
|
| 592 |
+
- id (INTEGER, PK) | order_id (FKβorders) | product_id (FKβproducts)
|
| 593 |
+
- quantity (INTEGER) | unit_price (REAL)
|
| 594 |
+
|
| 595 |
+
TABLE: returns
|
| 596 |
+
- id (INTEGER, PK) | order_item_id (FKβorder_items) | reason (TEXT)
|
| 597 |
+
- return_date (TEXT) | refund_amount (REAL)
|
| 598 |
+
- status (TEXT, 'pending'/'refunded'/'rejected')
|
| 599 |
+
|
| 600 |
+
RELATIONSHIPS:
|
| 601 |
+
products.category_id β categories.id
|
| 602 |
+
products.seller_id β sellers.id
|
| 603 |
+
orders.user_id β users.id
|
| 604 |
+
order_items.order_id β orders.id
|
| 605 |
+
order_items.product_id β products.id
|
| 606 |
+
returns.order_item_id β order_items.id
|
| 607 |
+
|
| 608 |
+
NOTE: Categories have a parent_category for hierarchy (e.g., Smartphones β Electronics).
|
| 609 |
+
Returns reference order_items, not orders directly.
|
| 610 |
+
"""
|
| 611 |
+
|
| 612 |
+
|
| 613 |
+
# ---------------------------------------------------------------------------
|
| 614 |
+
# Schema registry
|
| 615 |
+
# ---------------------------------------------------------------------------
|
| 616 |
+
|
| 617 |
+
SCHEMAS: Dict[str, DatabaseSchema] = {
|
| 618 |
+
"company": DatabaseSchema(
|
| 619 |
+
id="company",
|
| 620 |
+
name="Company Analytics",
|
| 621 |
+
schema_sql=COMPANY_SCHEMA_SQL,
|
| 622 |
+
seed_sql=COMPANY_SEED_SQL,
|
| 623 |
+
description=COMPANY_DESCRIPTION,
|
| 624 |
+
),
|
| 625 |
+
"hospital": DatabaseSchema(
|
| 626 |
+
id="hospital",
|
| 627 |
+
name="Hospital Management",
|
| 628 |
+
schema_sql=HOSPITAL_SCHEMA_SQL,
|
| 629 |
+
seed_sql=HOSPITAL_SEED_SQL,
|
| 630 |
+
description=HOSPITAL_DESCRIPTION,
|
| 631 |
+
),
|
| 632 |
+
"ecommerce": DatabaseSchema(
|
| 633 |
+
id="ecommerce",
|
| 634 |
+
name="E-Commerce Platform",
|
| 635 |
+
schema_sql=ECOMMERCE_SCHEMA_SQL,
|
| 636 |
+
seed_sql=ECOMMERCE_SEED_SQL,
|
| 637 |
+
description=ECOMMERCE_DESCRIPTION,
|
| 638 |
+
),
|
| 639 |
+
}
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def create_database(schema_id: str = "company") -> sqlite3.Connection:
|
| 643 |
+
"""Create an in-memory SQLite database for a specific domain."""
|
| 644 |
+
schema = SCHEMAS[schema_id]
|
| 645 |
+
conn = sqlite3.connect(":memory:", check_same_thread=False)
|
| 646 |
+
conn.execute("PRAGMA foreign_keys = ON")
|
| 647 |
+
conn.executescript(schema.schema_sql)
|
| 648 |
+
conn.executescript(schema.seed_sql)
|
| 649 |
+
return conn
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def get_expected_result(conn: sqlite3.Connection, query: str) -> List[Tuple]:
|
| 653 |
+
"""Execute a query and return the result set."""
|
| 654 |
+
cursor = conn.execute(query)
|
| 655 |
+
return cursor.fetchall()
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
def get_column_names(conn: sqlite3.Connection, query: str) -> List[str]:
|
| 659 |
+
"""Get column names from a query result."""
|
| 660 |
+
cursor = conn.execute(query)
|
| 661 |
+
return [desc[0] for desc in cursor.description] if cursor.description else []
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
# ---------------------------------------------------------------------------
|
| 665 |
+
# Company tasks
|
| 666 |
+
# ---------------------------------------------------------------------------
|
| 667 |
+
|
| 668 |
+
COMPANY_TASKS = [
|
| 669 |
+
# ββ EASY ββ
|
| 670 |
+
Task(
|
| 671 |
+
id="company_easy_1",
|
| 672 |
+
difficulty="easy",
|
| 673 |
+
schema_id="company",
|
| 674 |
+
question="List all active employees in the Engineering department, showing their name and salary, sorted by salary from highest to lowest.",
|
| 675 |
+
ground_truth_query="""
|
| 676 |
+
SELECT e.name, e.salary
|
| 677 |
+
FROM employees e
|
| 678 |
+
JOIN departments d ON e.department_id = d.id
|
| 679 |
+
WHERE d.name = 'Engineering' AND e.is_active = 1
|
| 680 |
+
ORDER BY e.salary DESC
|
| 681 |
+
""",
|
| 682 |
+
expected_columns=["name", "salary"],
|
| 683 |
+
description="Simple filtered query with sorting",
|
| 684 |
+
hints=["Use employees and departments tables", "Filter by department name and active status"],
|
| 685 |
+
max_steps=5,
|
| 686 |
+
),
|
| 687 |
+
Task(
|
| 688 |
+
id="company_easy_2",
|
| 689 |
+
difficulty="easy",
|
| 690 |
+
schema_id="company",
|
| 691 |
+
question="Show all products in the 'Electronics' category with a price greater than $50, sorted by price ascending.",
|
| 692 |
+
ground_truth_query="""
|
| 693 |
+
SELECT name, price
|
| 694 |
+
FROM products
|
| 695 |
+
WHERE category = 'Electronics' AND price > 50
|
| 696 |
+
ORDER BY price ASC
|
| 697 |
+
""",
|
| 698 |
+
expected_columns=["name", "price"],
|
| 699 |
+
description="Single table filter and sort",
|
| 700 |
+
hints=["Use the products table", "Filter by category and price"],
|
| 701 |
+
max_steps=5,
|
| 702 |
+
),
|
| 703 |
+
# ββ MEDIUM ββ
|
| 704 |
+
Task(
|
| 705 |
+
id="company_medium_1",
|
| 706 |
+
difficulty="medium",
|
| 707 |
+
schema_id="company",
|
| 708 |
+
question="Find the total revenue (sum of total_price) per product category for completed orders. Show the category and total revenue, sorted by revenue descending.",
|
| 709 |
+
ground_truth_query="""
|
| 710 |
+
SELECT p.category, SUM(o.total_price) AS total_revenue
|
| 711 |
+
FROM orders o
|
| 712 |
+
JOIN products p ON o.product_id = p.id
|
| 713 |
+
WHERE o.status = 'completed'
|
| 714 |
+
GROUP BY p.category
|
| 715 |
+
ORDER BY total_revenue DESC
|
| 716 |
+
""",
|
| 717 |
+
expected_columns=["category", "total_revenue"],
|
| 718 |
+
description="Aggregation with JOIN and GROUP BY",
|
| 719 |
+
hints=["Join orders with products", "Filter for completed orders", "Group by category"],
|
| 720 |
+
max_steps=6,
|
| 721 |
+
),
|
| 722 |
+
Task(
|
| 723 |
+
id="company_medium_2",
|
| 724 |
+
difficulty="medium",
|
| 725 |
+
schema_id="company",
|
| 726 |
+
question="Find the average salary per department, but only show departments where the average salary exceeds $100,000. Show department name and average salary, sorted by average salary descending.",
|
| 727 |
+
ground_truth_query="""
|
| 728 |
+
SELECT d.name, AVG(e.salary) AS avg_salary
|
| 729 |
+
FROM employees e
|
| 730 |
+
JOIN departments d ON e.department_id = d.id
|
| 731 |
+
WHERE e.is_active = 1
|
| 732 |
+
GROUP BY d.name
|
| 733 |
+
HAVING AVG(e.salary) > 100000
|
| 734 |
+
ORDER BY avg_salary DESC
|
| 735 |
+
""",
|
| 736 |
+
expected_columns=["name", "avg_salary"],
|
| 737 |
+
description="GROUP BY with HAVING clause",
|
| 738 |
+
hints=["Join employees with departments", "Use GROUP BY and HAVING"],
|
| 739 |
+
max_steps=6,
|
| 740 |
+
),
|
| 741 |
+
# ββ HARD ββ
|
| 742 |
+
Task(
|
| 743 |
+
id="company_hard_1",
|
| 744 |
+
difficulty="hard",
|
| 745 |
+
schema_id="company",
|
| 746 |
+
question="Calculate each department's salary budget utilization: show the department name, total active employee salaries, department budget, and the utilization percentage (total salaries / budget * 100, rounded to 1 decimal). Sort by utilization percentage descending.",
|
| 747 |
+
ground_truth_query="""
|
| 748 |
+
SELECT
|
| 749 |
+
d.name,
|
| 750 |
+
SUM(e.salary) AS total_salaries,
|
| 751 |
+
d.budget,
|
| 752 |
+
ROUND(SUM(e.salary) * 100.0 / d.budget, 1) AS utilization_pct
|
| 753 |
+
FROM departments d
|
| 754 |
+
JOIN employees e ON d.id = e.department_id
|
| 755 |
+
WHERE e.is_active = 1
|
| 756 |
+
GROUP BY d.id, d.name, d.budget
|
| 757 |
+
ORDER BY utilization_pct DESC
|
| 758 |
+
""",
|
| 759 |
+
expected_columns=["name", "total_salaries", "budget", "utilization_pct"],
|
| 760 |
+
description="Computed columns with percentage calculations",
|
| 761 |
+
hints=[
|
| 762 |
+
"Join departments with employees",
|
| 763 |
+
"Filter active employees only",
|
| 764 |
+
"Compute: total_salaries / budget * 100",
|
| 765 |
+
],
|
| 766 |
+
max_steps=8,
|
| 767 |
+
challenge_type="computation",
|
| 768 |
+
),
|
| 769 |
+
]
|
| 770 |
+
|
| 771 |
+
# ---------------------------------------------------------------------------
|
| 772 |
+
# Hospital tasks
|
| 773 |
+
# ---------------------------------------------------------------------------
|
| 774 |
+
|
| 775 |
+
HOSPITAL_TASKS = [
|
| 776 |
+
# ββ EASY ββ
|
| 777 |
+
Task(
|
| 778 |
+
id="hospital_easy_1",
|
| 779 |
+
difficulty="easy",
|
| 780 |
+
schema_id="hospital",
|
| 781 |
+
question="List all patients who are currently still admitted (discharge_date is NULL). Show their name, diagnosis, and admission date, sorted by admission date.",
|
| 782 |
+
ground_truth_query="""
|
| 783 |
+
SELECT name, diagnosis, admission_date
|
| 784 |
+
FROM patients
|
| 785 |
+
WHERE discharge_date IS NULL
|
| 786 |
+
ORDER BY admission_date ASC
|
| 787 |
+
""",
|
| 788 |
+
expected_columns=["name", "diagnosis", "admission_date"],
|
| 789 |
+
description="NULL handling filter",
|
| 790 |
+
hints=["Use the patients table", "NULL means still admitted β use IS NULL"],
|
| 791 |
+
max_steps=5,
|
| 792 |
+
challenge_type="null_handling",
|
| 793 |
+
),
|
| 794 |
+
# ββ MEDIUM ββ
|
| 795 |
+
Task(
|
| 796 |
+
id="hospital_medium_1",
|
| 797 |
+
difficulty="medium",
|
| 798 |
+
schema_id="hospital",
|
| 799 |
+
question="Find the total medication cost for each currently admitted patient. Show the patient name, number of active prescriptions (end_date IS NULL), and the total daily medication cost (sum of medication unit_cost). Sort by total cost descending.",
|
| 800 |
+
ground_truth_query="""
|
| 801 |
+
SELECT
|
| 802 |
+
p.name,
|
| 803 |
+
COUNT(pr.id) AS active_prescriptions,
|
| 804 |
+
SUM(m.unit_cost) AS daily_med_cost
|
| 805 |
+
FROM patients p
|
| 806 |
+
JOIN prescriptions pr ON p.id = pr.patient_id
|
| 807 |
+
JOIN medications m ON pr.medication_id = m.id
|
| 808 |
+
WHERE p.discharge_date IS NULL AND pr.end_date IS NULL
|
| 809 |
+
GROUP BY p.id, p.name
|
| 810 |
+
ORDER BY daily_med_cost DESC
|
| 811 |
+
""",
|
| 812 |
+
expected_columns=["name", "active_prescriptions", "daily_med_cost"],
|
| 813 |
+
description="Multi-table JOIN with NULL filters and aggregation",
|
| 814 |
+
hints=[
|
| 815 |
+
"Join patients, prescriptions, and medications",
|
| 816 |
+
"Filter: patient still admitted (discharge_date IS NULL)",
|
| 817 |
+
"Filter: prescription still active (end_date IS NULL)",
|
| 818 |
+
],
|
| 819 |
+
max_steps=7,
|
| 820 |
+
challenge_type="multi_null",
|
| 821 |
+
),
|
| 822 |
+
# ββ HARD ββ
|
| 823 |
+
Task(
|
| 824 |
+
id="hospital_hard_1",
|
| 825 |
+
difficulty="hard",
|
| 826 |
+
schema_id="hospital",
|
| 827 |
+
question="Find doctors who have prescribed medications to patients outside their own ward. Show the doctor's name, their ward name, the patient's name, the patient's ward name, and the medication prescribed. Sort by doctor name.",
|
| 828 |
+
ground_truth_query="""
|
| 829 |
+
SELECT
|
| 830 |
+
d.name AS doctor_name,
|
| 831 |
+
wd.name AS doctor_ward,
|
| 832 |
+
p.name AS patient_name,
|
| 833 |
+
wp.name AS patient_ward,
|
| 834 |
+
m.name AS medication
|
| 835 |
+
FROM prescriptions pr
|
| 836 |
+
JOIN doctors d ON pr.doctor_id = d.id
|
| 837 |
+
JOIN patients p ON pr.patient_id = p.id
|
| 838 |
+
JOIN medications m ON pr.medication_id = m.id
|
| 839 |
+
JOIN wards wd ON d.ward_id = wd.id
|
| 840 |
+
JOIN wards wp ON p.ward_id = wp.id
|
| 841 |
+
WHERE d.ward_id != p.ward_id
|
| 842 |
+
ORDER BY d.name
|
| 843 |
+
""",
|
| 844 |
+
expected_columns=["doctor_name", "doctor_ward", "patient_name", "patient_ward", "medication"],
|
| 845 |
+
description="Cross-ward prescription analysis β 5-table JOIN with self-referencing comparison",
|
| 846 |
+
hints=[
|
| 847 |
+
"Join prescriptions with doctors, patients, medications, and wards (twice!)",
|
| 848 |
+
"Join wards once for doctor's ward and once for patient's ward",
|
| 849 |
+
"Compare doctor's ward_id with patient's ward_id",
|
| 850 |
+
],
|
| 851 |
+
max_steps=10,
|
| 852 |
+
challenge_type="multi_join_complex",
|
| 853 |
+
),
|
| 854 |
+
]
|
| 855 |
+
|
| 856 |
+
# ---------------------------------------------------------------------------
|
| 857 |
+
# E-Commerce tasks
|
| 858 |
+
# ---------------------------------------------------------------------------
|
| 859 |
+
|
| 860 |
+
ECOMMERCE_TASKS = [
|
| 861 |
+
# ββ EASY ββ
|
| 862 |
+
Task(
|
| 863 |
+
id="ecommerce_easy_1",
|
| 864 |
+
difficulty="easy",
|
| 865 |
+
schema_id="ecommerce",
|
| 866 |
+
question="List all premium users and the city they are from, along with their signup date. Sort by signup date, most recent first.",
|
| 867 |
+
ground_truth_query="""
|
| 868 |
+
SELECT name, city, signup_date
|
| 869 |
+
FROM users
|
| 870 |
+
WHERE is_premium = 1
|
| 871 |
+
ORDER BY signup_date DESC
|
| 872 |
+
""",
|
| 873 |
+
expected_columns=["name", "city", "signup_date"],
|
| 874 |
+
description="Simple filter on single table",
|
| 875 |
+
hints=["Use the users table", "Filter by is_premium = 1"],
|
| 876 |
+
max_steps=5,
|
| 877 |
+
),
|
| 878 |
+
# ββ MEDIUM ββ
|
| 879 |
+
Task(
|
| 880 |
+
id="ecommerce_medium_1",
|
| 881 |
+
difficulty="medium",
|
| 882 |
+
schema_id="ecommerce",
|
| 883 |
+
question="Find the total revenue per seller for delivered orders only. Show the seller name, seller rating, number of items sold, and total revenue. Sort by total revenue descending.",
|
| 884 |
+
ground_truth_query="""
|
| 885 |
+
SELECT
|
| 886 |
+
s.name,
|
| 887 |
+
s.rating,
|
| 888 |
+
SUM(oi.quantity) AS items_sold,
|
| 889 |
+
SUM(oi.quantity * oi.unit_price) AS total_revenue
|
| 890 |
+
FROM sellers s
|
| 891 |
+
JOIN products p ON s.id = p.seller_id
|
| 892 |
+
JOIN order_items oi ON p.id = oi.product_id
|
| 893 |
+
JOIN orders o ON oi.order_id = o.id
|
| 894 |
+
WHERE o.status = 'delivered'
|
| 895 |
+
GROUP BY s.id, s.name, s.rating
|
| 896 |
+
ORDER BY total_revenue DESC
|
| 897 |
+
""",
|
| 898 |
+
expected_columns=["name", "rating", "items_sold", "total_revenue"],
|
| 899 |
+
description="4-table JOIN with aggregation and status filter",
|
| 900 |
+
hints=[
|
| 901 |
+
"Chain: sellers β products β order_items β orders",
|
| 902 |
+
"Filter for delivered orders",
|
| 903 |
+
"Group by seller",
|
| 904 |
+
],
|
| 905 |
+
max_steps=7,
|
| 906 |
+
),
|
| 907 |
+
# ββ HARD ββ
|
| 908 |
+
Task(
|
| 909 |
+
id="ecommerce_hard_1",
|
| 910 |
+
difficulty="hard",
|
| 911 |
+
schema_id="ecommerce",
|
| 912 |
+
question="Calculate the return rate for each product category. Show the category name, total items sold (from delivered orders), total items returned, and the return rate as a percentage (rounded to 1 decimal). Include categories with zero returns. Sort by return rate descending.",
|
| 913 |
+
ground_truth_query="""
|
| 914 |
+
SELECT
|
| 915 |
+
c.name AS category,
|
| 916 |
+
COALESCE(SUM(oi.quantity), 0) AS total_sold,
|
| 917 |
+
COALESCE(returned.total_returned, 0) AS total_returned,
|
| 918 |
+
ROUND(
|
| 919 |
+
COALESCE(returned.total_returned, 0) * 100.0 /
|
| 920 |
+
CASE WHEN SUM(oi.quantity) = 0 THEN 1 ELSE SUM(oi.quantity) END,
|
| 921 |
+
1
|
| 922 |
+
) AS return_rate_pct
|
| 923 |
+
FROM categories c
|
| 924 |
+
JOIN products p ON c.id = p.category_id
|
| 925 |
+
JOIN order_items oi ON p.id = oi.product_id
|
| 926 |
+
JOIN orders o ON oi.order_id = o.id
|
| 927 |
+
LEFT JOIN (
|
| 928 |
+
SELECT c2.id AS cat_id, COUNT(r.id) AS total_returned
|
| 929 |
+
FROM returns r
|
| 930 |
+
JOIN order_items oi2 ON r.order_item_id = oi2.id
|
| 931 |
+
JOIN products p2 ON oi2.product_id = p2.id
|
| 932 |
+
JOIN categories c2 ON p2.category_id = c2.id
|
| 933 |
+
GROUP BY c2.id
|
| 934 |
+
) returned ON c.id = returned.cat_id
|
| 935 |
+
WHERE o.status = 'delivered'
|
| 936 |
+
GROUP BY c.id, c.name, returned.total_returned
|
| 937 |
+
ORDER BY return_rate_pct DESC
|
| 938 |
+
""",
|
| 939 |
+
expected_columns=["category", "total_sold", "total_returned", "return_rate_pct"],
|
| 940 |
+
description="Return rate analysis β subquery + LEFT JOIN + COALESCE + division-by-zero handling",
|
| 941 |
+
hints=[
|
| 942 |
+
"You need to count items sold per category AND items returned per category",
|
| 943 |
+
"Use a subquery or LEFT JOIN for returns (some categories have zero returns)",
|
| 944 |
+
"Handle division by zero with CASE WHEN",
|
| 945 |
+
"Use COALESCE for NULL return counts",
|
| 946 |
+
],
|
| 947 |
+
max_steps=10,
|
| 948 |
+
challenge_type="subquery_with_edge_cases",
|
| 949 |
+
),
|
| 950 |
+
]
|
| 951 |
+
|
| 952 |
+
|
| 953 |
+
# ---------------------------------------------------------------------------
|
| 954 |
+
# Expert tasks β designed to genuinely challenge frontier models
|
| 955 |
+
# ---------------------------------------------------------------------------
|
| 956 |
+
|
| 957 |
+
EXPERT_TASKS = [
|
| 958 |
+
# Self-join: employees vs their managers (company)
|
| 959 |
+
Task(
|
| 960 |
+
id="company_hard_2",
|
| 961 |
+
difficulty="hard",
|
| 962 |
+
schema_id="company",
|
| 963 |
+
question="Find employees who earn more than their direct manager. Show the employee name, employee salary, manager name, and manager salary. Only include active employees with active managers. Sort by the salary difference (employee salary minus manager salary) descending.",
|
| 964 |
+
ground_truth_query="""
|
| 965 |
+
SELECT
|
| 966 |
+
e.name AS employee_name,
|
| 967 |
+
e.salary AS employee_salary,
|
| 968 |
+
m.name AS manager_name,
|
| 969 |
+
m.salary AS manager_salary
|
| 970 |
+
FROM employees e
|
| 971 |
+
JOIN employees m ON e.manager_id = m.id
|
| 972 |
+
WHERE e.is_active = 1 AND m.is_active = 1
|
| 973 |
+
AND e.salary > m.salary
|
| 974 |
+
ORDER BY (e.salary - m.salary) DESC
|
| 975 |
+
""",
|
| 976 |
+
expected_columns=["employee_name", "employee_salary", "manager_name", "manager_salary"],
|
| 977 |
+
description="Self-join with comparison across hierarchical relationship",
|
| 978 |
+
hints=[
|
| 979 |
+
"You need to join the employees table with itself",
|
| 980 |
+
"Alias one as 'e' (employee) and the other as 'm' (manager)",
|
| 981 |
+
"The manager_id column links employee to manager",
|
| 982 |
+
"Compare salaries between the employee row and the manager row",
|
| 983 |
+
],
|
| 984 |
+
max_steps=8,
|
| 985 |
+
challenge_type="self_join",
|
| 986 |
+
),
|
| 987 |
+
|
| 988 |
+
# Temporal reasoning: readmission within 30 days (hospital)
|
| 989 |
+
Task(
|
| 990 |
+
id="hospital_hard_2",
|
| 991 |
+
difficulty="hard",
|
| 992 |
+
schema_id="hospital",
|
| 993 |
+
question="Find patients who had multiple appointments with the same doctor. Show the patient name, doctor name, number of appointments together, and the date range (earliest to latest appointment date). Only count completed appointments. Sort by number of appointments descending.",
|
| 994 |
+
ground_truth_query="""
|
| 995 |
+
SELECT
|
| 996 |
+
p.name AS patient_name,
|
| 997 |
+
d.name AS doctor_name,
|
| 998 |
+
COUNT(a.id) AS visit_count,
|
| 999 |
+
MIN(a.appointment_date) AS first_visit,
|
| 1000 |
+
MAX(a.appointment_date) AS last_visit
|
| 1001 |
+
FROM appointments a
|
| 1002 |
+
JOIN patients p ON a.patient_id = p.id
|
| 1003 |
+
JOIN doctors d ON a.doctor_id = d.id
|
| 1004 |
+
WHERE a.status = 'completed'
|
| 1005 |
+
GROUP BY p.id, p.name, d.id, d.name
|
| 1006 |
+
HAVING COUNT(a.id) > 1
|
| 1007 |
+
ORDER BY visit_count DESC
|
| 1008 |
+
""",
|
| 1009 |
+
expected_columns=["patient_name", "doctor_name", "visit_count", "first_visit", "last_visit"],
|
| 1010 |
+
description="Temporal analysis with GROUP BY HAVING on joined data",
|
| 1011 |
+
hints=[
|
| 1012 |
+
"Join appointments with patients and doctors",
|
| 1013 |
+
"Group by patient-doctor pairs",
|
| 1014 |
+
"Use COUNT to find pairs with more than one appointment",
|
| 1015 |
+
"Use MIN/MAX for date range",
|
| 1016 |
+
],
|
| 1017 |
+
max_steps=8,
|
| 1018 |
+
challenge_type="temporal_grouping",
|
| 1019 |
+
),
|
| 1020 |
+
|
| 1021 |
+
# Multi-step: products in stock but never ordered (ecommerce)
|
| 1022 |
+
Task(
|
| 1023 |
+
id="ecommerce_hard_2",
|
| 1024 |
+
difficulty="hard",
|
| 1025 |
+
schema_id="ecommerce",
|
| 1026 |
+
question="Find products that are currently in stock (stock > 0) but have never been ordered. Show the product name, category name, seller name, price, and stock count. Sort by stock count descending. This helps identify inventory that isn't moving.",
|
| 1027 |
+
ground_truth_query="""
|
| 1028 |
+
SELECT
|
| 1029 |
+
p.name AS product_name,
|
| 1030 |
+
c.name AS category_name,
|
| 1031 |
+
s.name AS seller_name,
|
| 1032 |
+
p.price,
|
| 1033 |
+
p.stock
|
| 1034 |
+
FROM products p
|
| 1035 |
+
JOIN categories c ON p.category_id = c.id
|
| 1036 |
+
JOIN sellers s ON p.seller_id = s.id
|
| 1037 |
+
LEFT JOIN order_items oi ON p.id = oi.product_id
|
| 1038 |
+
WHERE oi.id IS NULL AND p.stock > 0
|
| 1039 |
+
ORDER BY p.stock DESC
|
| 1040 |
+
""",
|
| 1041 |
+
expected_columns=["product_name", "category_name", "seller_name", "price", "stock"],
|
| 1042 |
+
description="LEFT JOIN with IS NULL to find non-matching rows",
|
| 1043 |
+
hints=[
|
| 1044 |
+
"Use LEFT JOIN between products and order_items",
|
| 1045 |
+
"A product with no orders will have NULL in the order_items columns",
|
| 1046 |
+
"Filter with WHERE oi.id IS NULL to find products never ordered",
|
| 1047 |
+
"Also filter for stock > 0",
|
| 1048 |
+
],
|
| 1049 |
+
max_steps=8,
|
| 1050 |
+
challenge_type="anti_join",
|
| 1051 |
+
),
|
| 1052 |
+
]
|
| 1053 |
+
|
| 1054 |
+
|
| 1055 |
+
# ---------------------------------------------------------------------------
|
| 1056 |
+
# All tasks combined
|
| 1057 |
+
# ---------------------------------------------------------------------------
|
| 1058 |
+
|
| 1059 |
+
ALL_TASKS_LIST = COMPANY_TASKS + HOSPITAL_TASKS + ECOMMERCE_TASKS + EXPERT_TASKS
|
| 1060 |
+
|
| 1061 |
+
ALL_TASKS = {
|
| 1062 |
+
"easy": [t for t in ALL_TASKS_LIST if t.difficulty == "easy"],
|
| 1063 |
+
"medium": [t for t in ALL_TASKS_LIST if t.difficulty == "medium"],
|
| 1064 |
+
"hard": [t for t in ALL_TASKS_LIST if t.difficulty == "hard"],
|
| 1065 |
+
}
|
| 1066 |
+
|
| 1067 |
+
TASK_MAP: Dict[str, Task] = {t.id: t for t in ALL_TASKS_LIST}
|
| 1068 |
+
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|