Spaces:
Sleeping
Sleeping
Download inference.py from Akhilllll/data_quality_agent: direct link, hf CLI and curl.
- Browser
- Download file 8.77 kB
-
https://huggingface.co/spaces/Akhilllll/data_quality_agent/resolve/main/inference.py
- Command line
-
hf download hf://spaces/Akhilllll/data_quality_agent/inference.py
-
curl -L -o inference.py https://huggingface.co/spaces/Akhilllll/data_quality_agent/resolve/main/inference.py
8.77 kB
| """ | |
| Inference script for Data Quality Agent Environment. | |
| Uses HuggingFace InferenceClient (free tier) to drive an LLM agent | |
| through a data-quality audit: inspect, diagnose, fix, and report. | |
| """ | |
| import asyncio | |
| import os | |
| import sys | |
| from typing import List | |
| # Allow running from the project directory | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from huggingface_hub import InferenceClient | |
| try: | |
| from data_quality_agent import DataQualityAgentAction, DataQualityAgentEnv | |
| except ModuleNotFoundError: | |
| from models import DataQualityAgentAction | |
| from client import DataQualityAgentEnv | |
| # --------------------------------------------------------------------------- | |
| # Config | |
| # --------------------------------------------------------------------------- | |
| MODEL_NAME = os.environ.get("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct") | |
| API_KEY = os.environ.get("HF_TOKEN", "") | |
| IMAGE_NAME = os.environ.get("IMAGE_NAME", "data_quality_agent-env:latest") | |
| BENCHMARK = "data_quality_agent" | |
| TASKS = [ | |
| {"name": "messy_csv", "seed": 100, "label": "Easy"}, | |
| {"name": "noisy_labels", "seed": 200, "label": "Medium"}, | |
| {"name": "production_drift", "seed": 300, "label": "Hard"}, | |
| ] | |
| MAX_STEPS = 40 | |
| MAX_TOTAL_REWARD = 40.0 | |
| SUCCESS_SCORE_THRESHOLD = 0.40 | |
| TEMPERATURE = 0.3 | |
| MAX_TOKENS = 600 | |
| # --------------------------------------------------------------------------- | |
| # System prompt - expert data engineer persona | |
| # --------------------------------------------------------------------------- | |
| SYSTEM_PROMPT = """\ | |
| You are an expert ML Data Engineer performing a data quality audit. | |
| WORKFLOW (follow this order): | |
| 1. DESCRIBE β get dataset overview | |
| 2. CHECK_MISSING β find missing values | |
| 3. CHECK_DUPLICATES β find duplicate rows | |
| 4. CHECK_OUTLIERS β find outliers in numeric columns | |
| 5. CHECK_LABELS β verify label quality and class balance | |
| 6. CHECK_DISTRIBUTION <col> β check for distribution drift (numeric cols) | |
| 7. INSPECT <col> β deep-dive suspicious columns | |
| 8. Apply fixes for every issue found: | |
| - Missing values: FIX_MISSING <col> median (numeric) or FIX_MISSING <col> mode (categorical) | |
| - Duplicates: REMOVE_DUPLICATES | |
| - Outliers: FIX_OUTLIERS <col> winsorize | |
| - Label noise: FIX_LABELS confidence_filter | |
| - Class imbalance: REBALANCE smote | |
| - Type issues: FIX_TYPES <col> | |
| - Distribution drift: FIX_DISTRIBUTION <col> | |
| 9. VALIDATE β check your progress | |
| 10. SUBMIT_REPORT β finalize | |
| CRITICAL RULES: | |
| - ALWAYS start with DESCRIBE in step 1 | |
| - Run ALL check commands before fixing anything | |
| - Use semicolons to run multiple commands per step (efficient!) | |
| - After fixing, always VALIDATE then SUBMIT_REPORT | |
| - Never skip SUBMIT_REPORT β it is required for full score | |
| - Choose the BEST method for each fix (see recommendations above) | |
| - Handle EVERY detected issue, not just some | |
| OUTPUT FORMAT: | |
| Respond ONLY with commands, semicolon-separated. No explanations. | |
| Example: CHECK_MISSING; CHECK_DUPLICATES; CHECK_OUTLIERS | |
| """ | |
| # --------------------------------------------------------------------------- | |
| # Logging | |
| # --------------------------------------------------------------------------- | |
| def log_start(task: str, env: str, model: str) -> None: | |
| print(f"[START] task={task} env={env} model={model}", flush=True) | |
| def log_step(step: int, action: str, reward: float, done: bool) -> None: | |
| print(f"[STEP] step={step} action={action!r} reward={reward} done={done}", flush=True) | |
| def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None: | |
| print(f"[END] success={success} steps={steps} score={score:.4f} rewards={rewards}", flush=True) | |
| # --------------------------------------------------------------------------- | |
| # LLM helper with retry | |
| # --------------------------------------------------------------------------- | |
| VALID_CMDS = [ | |
| "DESCRIBE", "INSPECT", "CHECK_MISSING", "CHECK_DUPLICATES", | |
| "CHECK_OUTLIERS", "CHECK_LABELS", "CHECK_DISTRIBUTION", | |
| "CHECK_CORRELATION", "FIX_MISSING", "FIX_OUTLIERS", | |
| "REMOVE_DUPLICATES", "FIX_LABELS", "REBALANCE", "FIX_TYPES", | |
| "FIX_DISTRIBUTION", "VALIDATE", "SUBMIT_REPORT", | |
| ] | |
| def get_model_message( | |
| client: InferenceClient, | |
| step: int, | |
| dashboard: str, | |
| last_reward: float, | |
| history: List[str], | |
| ) -> str: | |
| recent = history[-6:] if len(history) > 6 else history | |
| history_text = "\n".join(recent) if recent else "(audit just started)" | |
| user_prompt = ( | |
| f"STEP {step}/{MAX_STEPS} | Last reward: {last_reward:+.4f}\n\n" | |
| f"DASHBOARD:\n{dashboard}\n\n" | |
| f"RECENT ACTIONS:\n{history_text}\n\n" | |
| f"What commands do you issue next?" | |
| ) | |
| text = None | |
| for attempt in range(3): | |
| try: | |
| completion = client.chat.completions.create( | |
| messages=[ | |
| {"role": "system", "content": SYSTEM_PROMPT}, | |
| {"role": "user", "content": user_prompt}, | |
| ], | |
| temperature=TEMPERATURE, | |
| max_tokens=MAX_TOKENS, | |
| ) | |
| text = completion.choices[0].message["content"].strip() | |
| break | |
| except Exception as exc: | |
| print(f"[DEBUG] Retry {attempt+1} failed: {exc}", flush=True) | |
| if not text: | |
| return "DESCRIBE" | |
| # Extract valid commands | |
| lines = text.replace("\n", ";").split(";") | |
| cmd_lines = [] | |
| for line in lines: | |
| line = line.strip() | |
| if line and any(line.upper().startswith(cmd) for cmd in VALID_CMDS): | |
| cmd_lines.append(line) | |
| if cmd_lines: | |
| return "; ".join(cmd_lines) | |
| return "DESCRIBE" | |
| # --------------------------------------------------------------------------- | |
| # Main loop | |
| # --------------------------------------------------------------------------- | |
| async def run_task(client: InferenceClient, task_cfg: dict) -> float: | |
| task_name = task_cfg["name"] | |
| seed = task_cfg["seed"] | |
| env = DataQualityAgentEnv(base_url="http://localhost:8000") | |
| history: List[str] = [] | |
| rewards: List[float] = [] | |
| steps_taken = 0 | |
| score = 0.0 | |
| log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME) | |
| try: | |
| result = await env.reset(seed=seed) | |
| last_dashboard = result.observation.dashboard | |
| last_reward = 0.0 | |
| for step in range(1, MAX_STEPS + 1): | |
| if result.done: | |
| break | |
| message = get_model_message( | |
| client, step, last_dashboard, last_reward, history | |
| ) | |
| result = await env.step(DataQualityAgentAction(message=message)) | |
| reward = result.reward or 0.0 | |
| done = result.done | |
| rewards.append(reward) | |
| steps_taken = step | |
| last_dashboard = result.observation.dashboard | |
| last_reward = reward | |
| log_step(step, message, reward, done) | |
| history.append(f"Step {step}: {message!r} -> reward {reward:+.4f}") | |
| if done: | |
| break | |
| # The final step reward IS the graded score (0-1) from the environment | |
| # Use the last reward (which contains the final graded score) as the primary score | |
| # Also factor in per-step rewards for a richer signal | |
| if rewards: | |
| final_reward = rewards[-1] # final graded score from environment | |
| step_rewards = sum(rewards[:-1]) if len(rewards) > 1 else 0.0 | |
| # Weighted: 80% final grade + 20% step progress (normalized) | |
| step_component = min(1.0, step_rewards / 2.0) # cap step rewards | |
| score = final_reward * 0.80 + step_component * 0.20 | |
| else: | |
| score = 0.0 | |
| score = min(max(score, 0.0), 1.0) | |
| finally: | |
| try: | |
| await env.close() | |
| except Exception as e: | |
| print(f"[DEBUG] env.close() error: {e}", flush=True) | |
| log_end(score >= SUCCESS_SCORE_THRESHOLD, steps_taken, score, rewards) | |
| return score | |
| # --------------------------------------------------------------------------- | |
| # Entry | |
| # --------------------------------------------------------------------------- | |
| async def main() -> None: | |
| if not API_KEY: | |
| raise ValueError("HF_TOKEN not set. Export HF_TOKEN=<your token>") | |
| client = InferenceClient( | |
| model=MODEL_NAME, | |
| token=API_KEY, | |
| ) | |
| scores = {} | |
| for task_cfg in TASKS: | |
| s = await run_task(client, task_cfg) | |
| scores[task_cfg["name"]] = s | |
| print(f"\n{'='*50}") | |
| print(f"Task {task_cfg['label']} ({task_cfg['name']}): score={s:.4f}") | |
| print(f"{'='*50}\n") | |
| print("\n=== FINAL SCORES ===") | |
| for name, s in scores.items(): | |
| print(f" {name}: {s:.4f}") | |
| avg = sum(scores.values()) / len(scores) | |
| print(f" AVERAGE: {avg:.4f}") | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |