data_quality_agent / inference.py
Akhilllll's picture
Upload folder using huggingface_hub
6f12524 verified
Raw History Blame Contribute Delete
8.77 kB
"""
Inference script for Data Quality Agent Environment.
Uses HuggingFace InferenceClient (free tier) to drive an LLM agent
through a data-quality audit: inspect, diagnose, fix, and report.
"""
import asyncio
import os
import sys
from typing import List
# Allow running from the project directory
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from huggingface_hub import InferenceClient
try:
from data_quality_agent import DataQualityAgentAction, DataQualityAgentEnv
except ModuleNotFoundError:
from models import DataQualityAgentAction
from client import DataQualityAgentEnv
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
MODEL_NAME = os.environ.get("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")
API_KEY = os.environ.get("HF_TOKEN", "")
IMAGE_NAME = os.environ.get("IMAGE_NAME", "data_quality_agent-env:latest")
BENCHMARK = "data_quality_agent"
TASKS = [
{"name": "messy_csv", "seed": 100, "label": "Easy"},
{"name": "noisy_labels", "seed": 200, "label": "Medium"},
{"name": "production_drift", "seed": 300, "label": "Hard"},
]
MAX_STEPS = 40
MAX_TOTAL_REWARD = 40.0
SUCCESS_SCORE_THRESHOLD = 0.40
TEMPERATURE = 0.3
MAX_TOKENS = 600
# ---------------------------------------------------------------------------
# System prompt - expert data engineer persona
# ---------------------------------------------------------------------------
SYSTEM_PROMPT = """\
You are an expert ML Data Engineer performing a data quality audit.
WORKFLOW (follow this order):
1. DESCRIBE β€” get dataset overview
2. CHECK_MISSING β€” find missing values
3. CHECK_DUPLICATES β€” find duplicate rows
4. CHECK_OUTLIERS β€” find outliers in numeric columns
5. CHECK_LABELS β€” verify label quality and class balance
6. CHECK_DISTRIBUTION <col> β€” check for distribution drift (numeric cols)
7. INSPECT <col> β€” deep-dive suspicious columns
8. Apply fixes for every issue found:
- Missing values: FIX_MISSING <col> median (numeric) or FIX_MISSING <col> mode (categorical)
- Duplicates: REMOVE_DUPLICATES
- Outliers: FIX_OUTLIERS <col> winsorize
- Label noise: FIX_LABELS confidence_filter
- Class imbalance: REBALANCE smote
- Type issues: FIX_TYPES <col>
- Distribution drift: FIX_DISTRIBUTION <col>
9. VALIDATE β€” check your progress
10. SUBMIT_REPORT β€” finalize
CRITICAL RULES:
- ALWAYS start with DESCRIBE in step 1
- Run ALL check commands before fixing anything
- Use semicolons to run multiple commands per step (efficient!)
- After fixing, always VALIDATE then SUBMIT_REPORT
- Never skip SUBMIT_REPORT β€” it is required for full score
- Choose the BEST method for each fix (see recommendations above)
- Handle EVERY detected issue, not just some
OUTPUT FORMAT:
Respond ONLY with commands, semicolon-separated. No explanations.
Example: CHECK_MISSING; CHECK_DUPLICATES; CHECK_OUTLIERS
"""
# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------
def log_start(task: str, env: str, model: str) -> None:
print(f"[START] task={task} env={env} model={model}", flush=True)
def log_step(step: int, action: str, reward: float, done: bool) -> None:
print(f"[STEP] step={step} action={action!r} reward={reward} done={done}", flush=True)
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
print(f"[END] success={success} steps={steps} score={score:.4f} rewards={rewards}", flush=True)
# ---------------------------------------------------------------------------
# LLM helper with retry
# ---------------------------------------------------------------------------
VALID_CMDS = [
"DESCRIBE", "INSPECT", "CHECK_MISSING", "CHECK_DUPLICATES",
"CHECK_OUTLIERS", "CHECK_LABELS", "CHECK_DISTRIBUTION",
"CHECK_CORRELATION", "FIX_MISSING", "FIX_OUTLIERS",
"REMOVE_DUPLICATES", "FIX_LABELS", "REBALANCE", "FIX_TYPES",
"FIX_DISTRIBUTION", "VALIDATE", "SUBMIT_REPORT",
]
def get_model_message(
client: InferenceClient,
step: int,
dashboard: str,
last_reward: float,
history: List[str],
) -> str:
recent = history[-6:] if len(history) > 6 else history
history_text = "\n".join(recent) if recent else "(audit just started)"
user_prompt = (
f"STEP {step}/{MAX_STEPS} | Last reward: {last_reward:+.4f}\n\n"
f"DASHBOARD:\n{dashboard}\n\n"
f"RECENT ACTIONS:\n{history_text}\n\n"
f"What commands do you issue next?"
)
text = None
for attempt in range(3):
try:
completion = client.chat.completions.create(
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_prompt},
],
temperature=TEMPERATURE,
max_tokens=MAX_TOKENS,
)
text = completion.choices[0].message["content"].strip()
break
except Exception as exc:
print(f"[DEBUG] Retry {attempt+1} failed: {exc}", flush=True)
if not text:
return "DESCRIBE"
# Extract valid commands
lines = text.replace("\n", ";").split(";")
cmd_lines = []
for line in lines:
line = line.strip()
if line and any(line.upper().startswith(cmd) for cmd in VALID_CMDS):
cmd_lines.append(line)
if cmd_lines:
return "; ".join(cmd_lines)
return "DESCRIBE"
# ---------------------------------------------------------------------------
# Main loop
# ---------------------------------------------------------------------------
async def run_task(client: InferenceClient, task_cfg: dict) -> float:
task_name = task_cfg["name"]
seed = task_cfg["seed"]
env = DataQualityAgentEnv(base_url="http://localhost:8000")
history: List[str] = []
rewards: List[float] = []
steps_taken = 0
score = 0.0
log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)
try:
result = await env.reset(seed=seed)
last_dashboard = result.observation.dashboard
last_reward = 0.0
for step in range(1, MAX_STEPS + 1):
if result.done:
break
message = get_model_message(
client, step, last_dashboard, last_reward, history
)
result = await env.step(DataQualityAgentAction(message=message))
reward = result.reward or 0.0
done = result.done
rewards.append(reward)
steps_taken = step
last_dashboard = result.observation.dashboard
last_reward = reward
log_step(step, message, reward, done)
history.append(f"Step {step}: {message!r} -> reward {reward:+.4f}")
if done:
break
# The final step reward IS the graded score (0-1) from the environment
# Use the last reward (which contains the final graded score) as the primary score
# Also factor in per-step rewards for a richer signal
if rewards:
final_reward = rewards[-1] # final graded score from environment
step_rewards = sum(rewards[:-1]) if len(rewards) > 1 else 0.0
# Weighted: 80% final grade + 20% step progress (normalized)
step_component = min(1.0, step_rewards / 2.0) # cap step rewards
score = final_reward * 0.80 + step_component * 0.20
else:
score = 0.0
score = min(max(score, 0.0), 1.0)
finally:
try:
await env.close()
except Exception as e:
print(f"[DEBUG] env.close() error: {e}", flush=True)
log_end(score >= SUCCESS_SCORE_THRESHOLD, steps_taken, score, rewards)
return score
# ---------------------------------------------------------------------------
# Entry
# ---------------------------------------------------------------------------
async def main() -> None:
if not API_KEY:
raise ValueError("HF_TOKEN not set. Export HF_TOKEN=<your token>")
client = InferenceClient(
model=MODEL_NAME,
token=API_KEY,
)
scores = {}
for task_cfg in TASKS:
s = await run_task(client, task_cfg)
scores[task_cfg["name"]] = s
print(f"\n{'='*50}")
print(f"Task {task_cfg['label']} ({task_cfg['name']}): score={s:.4f}")
print(f"{'='*50}\n")
print("\n=== FINAL SCORES ===")
for name, s in scores.items():
print(f" {name}: {s:.4f}")
avg = sum(scores.values()) / len(scores)
print(f" AVERAGE: {avg:.4f}")
if __name__ == "__main__":
asyncio.run(main())