sql-repair-env / openenv.yaml
ashoka0402
clean commit (removed token)
64ccc23
Raw History Blame Contribute Delete
7.78 kB
name: sql-repair-env
version: "1.0.0"
description: >
Interactive multi-turn SQL query repair environment.
An agent receives a broken SQL query alongside a plain-English intent and
a compact schema hint. Over multiple turns it can run partial queries to
probe for syntax errors, ask clarifying questions about the schema or
intent, and finally submit a repaired query that is evaluated against a
SQLite ground-truth execution.
Three difficulty levels (easy / medium / hard) scale across error type,
schema complexity, and the minimum number of turns required for an
optimal solution.
author: "sql-repair-team"
license: "MIT"
environment:
base_url: "http://localhost:8000"
reset_endpoint: "POST /reset"
step_endpoint: "POST /step"
state_endpoint: "GET /state"
health_endpoint: "GET /health"
grader_endpoint: "GET /grader"
tasks:
- id: easy_001
difficulty: easy
description: "Fix a syntax error in a single-table SELECT"
max_steps: 3
- id: easy_002
difficulty: easy
description: "Fix a missing comma in the SELECT column list"
max_steps: 3
- id: easy_003
difficulty: easy
description: "Fix a wrong aggregate function name"
max_steps: 3
- id: easy_004
difficulty: easy
description: "Fix an unquoted string literal"
max_steps: 3
- id: easy_005
difficulty: easy
description: "Fix a misused DISTINCT syntax"
max_steps: 3
- id: easy_006
difficulty: easy
description: "Fix a missing BY in ORDER BY"
max_steps: 3
- id: easy_007
difficulty: easy
description: "Fix an incomplete LIKE pattern"
max_steps: 3
- id: easy_008
difficulty: easy
description: "Fix a correlated aggregate in WHERE clause"
max_steps: 3
- id: easy_009
difficulty: easy
description: "Fix non-inclusive range boundary operators"
max_steps: 3
- id: easy_010
difficulty: easy
description: "Fix incorrect inactive user filter"
max_steps: 3
- id: easy_011
difficulty: easy
description: "Fix double-equals operator"
max_steps: 3
- id: easy_012
difficulty: easy
description: "Fix wrong aggregate function spelling"
max_steps: 3
- id: easy_013
difficulty: easy
description: "Fix LIMIT with parentheses"
max_steps: 3
- id: easy_014
difficulty: easy
description: "Fix malformed NOT condition"
max_steps: 3
- id: easy_015
difficulty: easy
description: "Fix missing BY in GROUP BY"
max_steps: 3
- id: medium_001
difficulty: medium
description: "Fix wrong join column (orders.id vs orders.user_id)"
max_steps: 5
- id: medium_002
difficulty: medium
description: "Fix wrong join column in aggregation query"
max_steps: 5
- id: medium_003
difficulty: medium
description: "Fix wrong subquery column in NOT IN"
max_steps: 5
- id: medium_004
difficulty: medium
description: "Fix missing ON clause in JOIN"
max_steps: 5
- id: medium_005
difficulty: medium
description: "Fix WHERE/GROUP BY ordering with cross-table filter"
max_steps: 5
- id: medium_006
difficulty: medium
description: "Fix wrong join key between orders and products"
max_steps: 5
- id: medium_007
difficulty: medium
description: "Fix wrong subquery column for products NOT IN"
max_steps: 5
- id: medium_008
difficulty: medium
description: "Fix missing join for cross-table GROUP BY"
max_steps: 5
- id: medium_009
difficulty: medium
description: "Fix WHERE used instead of HAVING for aggregate"
max_steps: 5
- id: medium_010
difficulty: medium
description: "Fix ORDER BY placed before WHERE"
max_steps: 5
- id: medium_011
difficulty: medium
description: "Fix missing BY in GROUP BY with subquery"
max_steps: 5
- id: medium_012
difficulty: medium
description: "Fix INNER JOIN when LEFT JOIN is needed"
max_steps: 5
- id: medium_013
difficulty: medium
description: "Fix COUNT without DISTINCT in category grouping"
max_steps: 5
- id: medium_014
difficulty: medium
description: "Fix AND logic for multi-month order filter"
max_steps: 5
- id: medium_015
difficulty: medium
description: "Fix SUM in WHERE instead of HAVING"
max_steps: 5
- id: hard_001
difficulty: hard
description: "Fix multi-table join not matching product on both user and product"
max_steps: 8
- id: hard_002
difficulty: hard
description: "Fix AVG filter in WHERE and missing GROUP BY"
max_steps: 8
- id: hard_003
difficulty: hard
description: "Fix wrong join key through orders to tags"
max_steps: 8
- id: hard_004
difficulty: hard
description: "Fix CTE alias used incorrectly in outer query"
max_steps: 8
- id: hard_005
difficulty: hard
description: "Fix wrong join key between products and reviews"
max_steps: 8
- id: hard_006
difficulty: hard
description: "Fix NOT IN subquery with unnecessary NULL guard"
max_steps: 8
- id: hard_007
difficulty: hard
description: "Fix AND logic on single tag row — requires self-join"
max_steps: 8
- id: hard_008
difficulty: hard
description: "Fix top-N per group without window functions"
max_steps: 8
- id: hard_009
difficulty: hard
description: "Fix nested aggregate — requires CTE"
max_steps: 8
- id: hard_010
difficulty: hard
description: "Fix multi-condition filter with city-level aggregation"
max_steps: 8
action_space:
- name: submit_query
description: "Submit the final repaired SQL query. Ends the episode."
fields:
- name: type
type: string
const: "submit_query"
- name: query
type: string
description: "The repaired SQL query string"
- name: run_partial
description: "Execute a query without finalising. Returns error message or OK status."
fields:
- name: type
type: string
const: "run_partial"
- name: query
type: string
description: "The SQL query to test"
- name: ask_clarification
description: "Ask a natural-language question about the schema or intent."
fields:
- name: type
type: string
const: "ask_clarification"
- name: question
type: string
description: "The clarification question"
observation_space:
- name: task_id
type: string
- name: difficulty
type: string
enum: [easy, medium, hard]
- name: intent
type: string
description: "Plain-English description of what the query should do"
- name: schema_hint
type: string
description: "Compact table signatures, e.g. 'users(id,name,age) | orders(id,user_id,amount)'"
- name: broken_query
type: string
description: "The original broken query (unchanged across turns)"
- name: turn
type: integer
- name: max_turns
type: integer
- name: error_message
type: string
nullable: true
description: "SQLite error from the last run_partial or failed submit"
- name: clarification_response
type: string
nullable: true
description: "Answer to the last ask_clarification action"
- name: last_action_type
type: string
nullable: true
enum: [submit_query, run_partial, ask_clarification]
reward:
range: [-1.0, 1.0]
description: >
submit_query: scored by Jaccard similarity of result vs expected rows,
minus a turn penalty of 0.05 per turn used.
run_partial: +0.05 on success, -0.10 on SQL error.
ask_clarification: 0.00 on hit, -0.05 on miss.
grader:
formula: "0.60 * correctness + 0.25 * turn_efficiency + 0.15 * clarification_quality"
range: [0.0, 1.0]
deterministic: true