""" Inference script for Data Quality Agent Environment. Uses HuggingFace InferenceClient (free tier) to drive an LLM agent through a data-quality audit: inspect, diagnose, fix, and report. """ import asyncio import os import sys from typing import List # Allow running from the project directory sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from huggingface_hub import InferenceClient try: from data_quality_agent import DataQualityAgentAction, DataQualityAgentEnv except ModuleNotFoundError: from models import DataQualityAgentAction from client import DataQualityAgentEnv # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- MODEL_NAME = os.environ.get("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct") API_KEY = os.environ.get("HF_TOKEN", "") IMAGE_NAME = os.environ.get("IMAGE_NAME", "data_quality_agent-env:latest") BENCHMARK = "data_quality_agent" TASKS = [ {"name": "messy_csv", "seed": 100, "label": "Easy"}, {"name": "noisy_labels", "seed": 200, "label": "Medium"}, {"name": "production_drift", "seed": 300, "label": "Hard"}, ] MAX_STEPS = 40 MAX_TOTAL_REWARD = 40.0 SUCCESS_SCORE_THRESHOLD = 0.40 TEMPERATURE = 0.3 MAX_TOKENS = 600 # --------------------------------------------------------------------------- # System prompt - expert data engineer persona # --------------------------------------------------------------------------- SYSTEM_PROMPT = """\ You are an expert ML Data Engineer performing a data quality audit. WORKFLOW (follow this order): 1. DESCRIBE — get dataset overview 2. CHECK_MISSING — find missing values 3. CHECK_DUPLICATES — find duplicate rows 4. CHECK_OUTLIERS — find outliers in numeric columns 5. CHECK_LABELS — verify label quality and class balance 6. CHECK_DISTRIBUTION — check for distribution drift (numeric cols) 7. INSPECT — deep-dive suspicious columns 8. Apply fixes for every issue found: - Missing values: FIX_MISSING median (numeric) or FIX_MISSING mode (categorical) - Duplicates: REMOVE_DUPLICATES - Outliers: FIX_OUTLIERS winsorize - Label noise: FIX_LABELS confidence_filter - Class imbalance: REBALANCE smote - Type issues: FIX_TYPES - Distribution drift: FIX_DISTRIBUTION 9. VALIDATE — check your progress 10. SUBMIT_REPORT — finalize CRITICAL RULES: - ALWAYS start with DESCRIBE in step 1 - Run ALL check commands before fixing anything - Use semicolons to run multiple commands per step (efficient!) - After fixing, always VALIDATE then SUBMIT_REPORT - Never skip SUBMIT_REPORT — it is required for full score - Choose the BEST method for each fix (see recommendations above) - Handle EVERY detected issue, not just some OUTPUT FORMAT: Respond ONLY with commands, semicolon-separated. No explanations. Example: CHECK_MISSING; CHECK_DUPLICATES; CHECK_OUTLIERS """ # --------------------------------------------------------------------------- # Logging # --------------------------------------------------------------------------- def log_start(task: str, env: str, model: str) -> None: print(f"[START] task={task} env={env} model={model}", flush=True) def log_step(step: int, action: str, reward: float, done: bool) -> None: print(f"[STEP] step={step} action={action!r} reward={reward} done={done}", flush=True) def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None: print(f"[END] success={success} steps={steps} score={score:.4f} rewards={rewards}", flush=True) # --------------------------------------------------------------------------- # LLM helper with retry # --------------------------------------------------------------------------- VALID_CMDS = [ "DESCRIBE", "INSPECT", "CHECK_MISSING", "CHECK_DUPLICATES", "CHECK_OUTLIERS", "CHECK_LABELS", "CHECK_DISTRIBUTION", "CHECK_CORRELATION", "FIX_MISSING", "FIX_OUTLIERS", "REMOVE_DUPLICATES", "FIX_LABELS", "REBALANCE", "FIX_TYPES", "FIX_DISTRIBUTION", "VALIDATE", "SUBMIT_REPORT", ] def get_model_message( client: InferenceClient, step: int, dashboard: str, last_reward: float, history: List[str], ) -> str: recent = history[-6:] if len(history) > 6 else history history_text = "\n".join(recent) if recent else "(audit just started)" user_prompt = ( f"STEP {step}/{MAX_STEPS} | Last reward: {last_reward:+.4f}\n\n" f"DASHBOARD:\n{dashboard}\n\n" f"RECENT ACTIONS:\n{history_text}\n\n" f"What commands do you issue next?" ) text = None for attempt in range(3): try: completion = client.chat.completions.create( messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user_prompt}, ], temperature=TEMPERATURE, max_tokens=MAX_TOKENS, ) text = completion.choices[0].message["content"].strip() break except Exception as exc: print(f"[DEBUG] Retry {attempt+1} failed: {exc}", flush=True) if not text: return "DESCRIBE" # Extract valid commands lines = text.replace("\n", ";").split(";") cmd_lines = [] for line in lines: line = line.strip() if line and any(line.upper().startswith(cmd) for cmd in VALID_CMDS): cmd_lines.append(line) if cmd_lines: return "; ".join(cmd_lines) return "DESCRIBE" # --------------------------------------------------------------------------- # Main loop # --------------------------------------------------------------------------- async def run_task(client: InferenceClient, task_cfg: dict) -> float: task_name = task_cfg["name"] seed = task_cfg["seed"] env = DataQualityAgentEnv(base_url="http://localhost:8000") history: List[str] = [] rewards: List[float] = [] steps_taken = 0 score = 0.0 log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME) try: result = await env.reset(seed=seed) last_dashboard = result.observation.dashboard last_reward = 0.0 for step in range(1, MAX_STEPS + 1): if result.done: break message = get_model_message( client, step, last_dashboard, last_reward, history ) result = await env.step(DataQualityAgentAction(message=message)) reward = result.reward or 0.0 done = result.done rewards.append(reward) steps_taken = step last_dashboard = result.observation.dashboard last_reward = reward log_step(step, message, reward, done) history.append(f"Step {step}: {message!r} -> reward {reward:+.4f}") if done: break # The final step reward IS the graded score (0-1) from the environment # Use the last reward (which contains the final graded score) as the primary score # Also factor in per-step rewards for a richer signal if rewards: final_reward = rewards[-1] # final graded score from environment step_rewards = sum(rewards[:-1]) if len(rewards) > 1 else 0.0 # Weighted: 80% final grade + 20% step progress (normalized) step_component = min(1.0, step_rewards / 2.0) # cap step rewards score = final_reward * 0.80 + step_component * 0.20 else: score = 0.0 score = min(max(score, 0.0), 1.0) finally: try: await env.close() except Exception as e: print(f"[DEBUG] env.close() error: {e}", flush=True) log_end(score >= SUCCESS_SCORE_THRESHOLD, steps_taken, score, rewards) return score # --------------------------------------------------------------------------- # Entry # --------------------------------------------------------------------------- async def main() -> None: if not API_KEY: raise ValueError("HF_TOKEN not set. Export HF_TOKEN=") client = InferenceClient( model=MODEL_NAME, token=API_KEY, ) scores = {} for task_cfg in TASKS: s = await run_task(client, task_cfg) scores[task_cfg["name"]] = s print(f"\n{'='*50}") print(f"Task {task_cfg['label']} ({task_cfg['name']}): score={s:.4f}") print(f"{'='*50}\n") print("\n=== FINAL SCORES ===") for name, s in scores.items(): print(f" {name}: {s:.4f}") avg = sum(scores.values()) / len(scores) print(f" AVERAGE: {avg:.4f}") if __name__ == "__main__": asyncio.run(main())