File size: 7,727 Bytes
64ccc23
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ed83e07
 
64ccc23
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
"""
app/models.py
=============
All Pydantic models for the Interactive SQL Query Repair environment.
These types are shared across env.py, grader.py, and the API server.
"""

from __future__ import annotations

from enum import Enum
from typing import Dict, List, Optional, Union

from pydantic import BaseModel, Field, field_validator


# ---------------------------------------------------------------------------
# Enums
# ---------------------------------------------------------------------------

class Difficulty(str, Enum):
    easy   = "easy"
    medium = "medium"
    hard   = "hard"


class ActionType(str, Enum):
    submit_query       = "submit_query"
    run_partial        = "run_partial"
    ask_clarification  = "ask_clarification"


class ErrorType(str, Enum):
    syntax  = "syntax"
    logic   = "logic"
    join    = "join"
    subquery = "subquery"
    ambiguous = "ambiguous"


# ---------------------------------------------------------------------------
# Dataset schema  (what lives in easy.json / medium.json / hard.json)
# ---------------------------------------------------------------------------

class SchemaInfo(BaseModel):
    """Human-readable table definitions shown to the agent."""
    tables: List[str] = Field(
        ...,
        description="Each entry is a table signature, e.g. 'users(id, name, age)'",
        examples=[["users(id, name, email, age)", "orders(id, user_id, amount, created_at)"]],
    )


class Task(BaseModel):
    """One repair task loaded from the dataset JSON files."""
    model_config = {"populate_by_name": True}

    id:                     str
    difficulty:             Difficulty
    schema:                 SchemaInfo = Field(..., alias="schema")
    intent:                 str   = Field(..., description="Plain-English goal for the query")
    broken_query:           str   = Field(..., description="The query the agent must fix")
    correct_query:          str   = Field(..., description="Reference solution (never shown to agent)")
    error_type:             ErrorType
    min_turns_expected:     int   = Field(..., ge=1)
    clarification_answers:  Dict[str, str] = Field(
        default_factory=dict,
        description="Keyword→answer map for ask_clarification actions",
    )

    @field_validator("min_turns_expected")
    @classmethod
    def turns_match_difficulty(cls, v: int, info) -> int:
        difficulty = info.data.get("difficulty")
        limits = {
            Difficulty.easy:   (1, 2),
            Difficulty.medium: (2, 4),
            Difficulty.hard:   (4, 8),
        }
        if difficulty in limits:
            lo, hi = limits[difficulty]
            if not (lo <= v <= hi):
                raise ValueError(
                    f"min_turns_expected={v} is out of range [{lo},{hi}] for difficulty={difficulty}"
                )
        return v


# ---------------------------------------------------------------------------
# Observation  (what the agent sees each turn)
# ---------------------------------------------------------------------------

class Observation(BaseModel):
    """
    Returned by reset() and every step().
    The agent never sees actual result rows — only error messages or
    execution status strings.
    """
    task_id:                  str
    difficulty:               Difficulty
    intent:                   str
    schema_hint:              str   = Field(..., description="Compact schema string, e.g. 'users(id,name,age)'")
    broken_query:             str   = Field(..., description="The original broken query (unchanged across turns)")
    turn:                     int   = Field(..., ge=0)
    max_turns:                int
    error_message:            Optional[str] = Field(
        None,
        description="Last execution error string; null on the very first turn",
    )
    clarification_response:   Optional[str] = Field(
        None,
        description="Answer to the last ask_clarification action; null otherwise",
    )
    last_action_type:         Optional[ActionType] = Field(
        None,
        description="Which action type was taken on the previous turn",
    )


# ---------------------------------------------------------------------------
# Actions  (what the agent can do each turn)
# ---------------------------------------------------------------------------

class SubmitQueryAction(BaseModel):
    type: ActionType = Field(ActionType.submit_query, frozen=True)
    query: str = Field(..., description="The agent's final repaired SQL query")


class RunPartialAction(BaseModel):
    type: ActionType = Field(ActionType.run_partial, frozen=True)
    query: str = Field(..., description="A query the agent wants to test without finalising")


class AskClarificationAction(BaseModel):
    type: ActionType = Field(ActionType.ask_clarification, frozen=True)
    question: str = Field(..., description="A natural-language question about the schema or intent")


# Union type accepted by POST /step
Action = Union[SubmitQueryAction, RunPartialAction, AskClarificationAction]


# ---------------------------------------------------------------------------
# Step result
# ---------------------------------------------------------------------------

class StepResult(BaseModel):
    observation:  Observation
    reward:       float  = Field(..., ge=-1.0, le=1.0)
    done:         bool
    info:         StepInfo


class StepInfo(BaseModel):
    """Extra diagnostics attached to every step result."""
    feedback:           str   = Field(..., description="Human-readable feedback string")
    rows_correct:       Optional[float] = Field(
        None,
        description="Jaccard similarity of result rows vs expected rows; only set on submit_query",
    )
    action_type:        ActionType
    turn:               int
    cumulative_reward:  float


# ---------------------------------------------------------------------------
# State  (GET /state)
# ---------------------------------------------------------------------------

class EnvState(BaseModel):
    task_id:        str
    difficulty:     Difficulty
    turn:           int
    max_turns:      int
    total_reward:   float
    done:           bool
    solved:         bool


# ---------------------------------------------------------------------------
# Grader output  (GET /grader)
# ---------------------------------------------------------------------------

class GraderResult(BaseModel):
    """
    Final evaluation score for a completed episode.
    score = 0.60 * correctness
          + 0.25 * turn_efficiency
          + 0.15 * clarification_quality
    All sub-scores and the final score are in [0.0, 1.0].
    """
    score:                   float = Field(..., ge=0.0, le=1.0)
    correctness:             float = Field(..., ge=0.0, le=1.0)
    turn_efficiency:         float = Field(..., ge=0.0, le=1.0)
    clarification_quality:   float = Field(..., ge=0.0, le=1.0)
    turns_used:              int
    max_turns:               int
    solved:                  bool
    detail:                  str   = Field(..., description="Short human-readable breakdown")


# ---------------------------------------------------------------------------
# API-level helpers
# ---------------------------------------------------------------------------

class ResetRequest(BaseModel):
    task_id:    str        = "easy_001"
    difficulty: Difficulty = Difficulty.easy


class TaskSummary(BaseModel):
    id:                 str
    difficulty:         Difficulty
    intent:             str
    error_type:         ErrorType
    min_turns_expected: int


class TaskListResponse(BaseModel):
    tasks: List[TaskSummary]


class HealthResponse(BaseModel):
    status: str = "ok"