File size: 7,441 Bytes
fee2f22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
import os
import requests
from openai import OpenAI
from typing import List, Optional

# Load .env file if present (for local development)
try:
    from dotenv import load_dotenv
    load_dotenv()
except ImportError:
    pass  # dotenv not required in production

# ── Config ──────────────────────────────────────────────────────────────
# Mandatory environment variables per hackathon requirements
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:7860")  # Default to local env
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")                # Default model
HF_TOKEN = os.getenv("HF_TOKEN")                                    # No default - required

# OpenAI client configured with environment variables
client = OpenAI(api_key=HF_TOKEN, base_url=API_BASE_URL)

ENV_URL = os.getenv("ENV_URL", "http://localhost:7860")
BENCHMARK = "data-cleaning-env"
TASKS = ["easy", "medium", "hard"]


# ── Structured Logging Functions (MANDATORY FORMAT) ─────────────────────
def log_start(task: str, env: str, model: str) -> None:
    """Emit [START] line at episode begin."""
    print(f"[START] task={task} env={env} model={model}", flush=True)


def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
    """Emit [STEP] line after each env.step() with exact format."""
    error_val = error if error else "null"
    done_val = str(done).lower()  # Must be lowercase: true/false
    print(
        f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",
        flush=True,
    )


def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
    """Emit [END] line after episode completes. Always called even on exception."""
    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
    success_val = str(success).lower()  # Must be lowercase: true/false
    print(
        f"[END] success={success_val} steps={steps} score={score:.3f} rewards={rewards_str}",
        flush=True,
    )


def ask_llm(observation: dict) -> dict:
    """Ask the LLM what cleaning action to take."""
    
    # Get first column as fallback
    columns = list(observation['current_data'].keys())
    first_column = columns[0] if columns else "id"
    
    prompt = f"""
You are a data cleaning agent. You receive a row of data and must clean it.

Current row data: {observation['current_data']}
Issues detected: {observation['issues_detected']}
Legal actions: {observation['legal_actions']}
Available columns: {columns}

Choose ONE action from the legal actions list.
Respond in this exact JSON format with ALL three fields always present:
{{
  "action_type": "fill_missing",
  "column": "age",
  "value": 30
}}

Rules:
- action_type must be from legal_actions list
- column must ALWAYS be one of: {columns}
- value is required for fill_missing, optional for others (use null if not needed)
- For fill_missing: provide a sensible default value
- For skip: still include column field using first available column
- ALWAYS include all three fields: action_type, column, value
"""
    response = client.chat.completions.create(
        model=MODEL_NAME,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.0,
    )

    import json
    text = response.choices[0].message.content.strip()
    text = text.replace("```json", "").replace("```", "").strip()
    parsed = json.loads(text)
    
    # Safety fallbacks
    if "column" not in parsed or not parsed["column"]:
        parsed["column"] = first_column
    if "value" not in parsed:
        parsed["value"] = None
    if parsed.get("column") not in columns:
        parsed["column"] = first_column
        
    return parsed


def run_task(task_name: str) -> float:
    """Run one full episode for a task and return score in [0, 1] range."""
    rewards: List[float] = []
    steps_taken = 0
    score = 0.0
    success = False
    
    log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)
    
    try:
        # Reset environment
        obs = requests.post(
            f"{ENV_URL}/reset",
            json={"task_name": task_name}
        ).json()

        # Run episode until done
        while not obs["done"]:
            steps_taken += 1
            
            # Get action from LLM
            error_msg = None
            try:
                action = ask_llm(obs)
                action_str = f"{action['action_type']}('{action['column']}',{action['value']})"
            except Exception as e:
                error_msg = str(e)
                # Fallback action on LLM error
                action = {
                    "action_type": "skip",
                    "column": list(obs["current_data"].keys())[0] if obs["current_data"] else "id",
                    "value": None
                }
                action_str = f"skip('{action['column']}',null)"

            # Take step in environment
            try:
                obs = requests.post(
                    f"{ENV_URL}/step",
                    json=action
                ).json()
                reward = obs.get("reward", 0.0)
                done = obs.get("done", False)
            except Exception as e:
                error_msg = str(e)
                reward = 0.0
                done = True
                obs = {"done": True, "reward": 0.0}
            
            rewards.append(reward)
            log_step(step=steps_taken, action=action_str, reward=reward, done=done, error=error_msg)
            
            if done:
                break

        # Get final score from environment grader (must be in [0, 1])
        try:
            state = requests.get(f"{ENV_URL}/state").json()
            score = state.get("score", 0.0)
            # Clamp score to [0, 1] range as required
            score = min(max(score, 0.0), 1.0)
            # Success if score meets reasonable threshold
            success = score >= 0.5
        except Exception as e:
            score = 0.0
            success = False

    except Exception as e:
        # Even on exception, we must emit [END]
        pass
    
    finally:
        # Always emit [END] line, even on exception
        log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
    
    return score


def main():
    """Main inference script entry point."""
    # Health check - verify server is running
    try:
        health = requests.get(f"{ENV_URL}/health", timeout=5).json()
        if health.get("status") != "ok":
            print(f"[ERROR] Server not healthy at {ENV_URL}", flush=True)
            return
    except Exception as e:
        print(f"[ERROR] Server not running at {ENV_URL}: {e}", flush=True)
        return

    # Run inference on all tasks
    scores = {}
    for task in TASKS:
        scores[task] = run_task(task)

    # Print summary (not part of structured output, just for convenience)
    print(f"\n{'='*60}", flush=True)
    print("BASELINE RESULTS SUMMARY", flush=True)
    print(f"{'='*60}", flush=True)
    for task, score in scores.items():
        print(f"  {task:10s}: {score:.3f}", flush=True)
    avg = sum(scores.values()) / len(scores) if scores else 0.0
    print(f"  {'average':10s}: {avg:.3f}", flush=True)
    print(f"{'='*60}", flush=True)


if __name__ == "__main__":
    main()