etl-debug-openenv / validate.py
AbhiDS16's picture
initial submission
eea03ca
Raw History Blame Contribute Delete
15.7 kB
"""
validate.py β€” Pre-submission validation for Data Pipeline Incident Response OpenEnv.
Checks:
[1] openenv.yaml exists and has required fields (including cascade task)
[2] All Python modules import cleanly
[3] All 4 tasks reset successfully and return valid Observations
[4] step() works and returns valid (Observation, Reward, done, info)
[5] New actions check_pipeline_health and audit_log work correctly
[6] finish action triggers grading and returns final_score in [0, 1]
[7] Grader scores are in [0.0, 1.0] and deterministic (run twice)
[8] State snapshot is non-empty and has required keys
[9] requirements.txt is non-empty
[10] Dockerfile exists and has required instructions
[11] inference.py has required variables and main()
[12] Task difficulty progression (cascade ≀ hard ≀ medium ≀ easy for do-nothing agent)
Run with:
python validate.py
"""
from __future__ import annotations
import importlib
import json
import sys
import traceback
from pathlib import Path
from typing import Any, Callable, Dict, List, Tuple
# ── Colour helpers ────────────────────────────────────────────────────────────
GREEN = "\033[92m"
RED = "\033[91m"
YELLOW = "\033[93m"
RESET = "\033[0m"
BOLD = "\033[1m"
PASS = f"{GREEN}βœ“ PASS{RESET}"
FAIL = f"{RED}βœ— FAIL{RESET}"
WARN = f"{YELLOW}⚠ WARN{RESET}"
def _ok(msg: str) -> None:
print(f" {PASS} {msg}")
def _fail(msg: str) -> None:
print(f" {FAIL} {msg}")
def _warn(msg: str) -> None:
print(f" {WARN} {msg}")
def _header(title: str) -> None:
print(f"\n{BOLD}{'─'*60}{RESET}")
print(f"{BOLD} {title}{RESET}")
print(f"{BOLD}{'─'*60}{RESET}")
# ── Individual checks ─────────────────────────────────────────────────────────
def check_yaml() -> bool:
_header("Check 1 β€” openenv.yaml")
path = Path("openenv.yaml")
if not path.exists():
_fail("openenv.yaml not found")
return False
try:
import yaml
data = yaml.safe_load(path.read_text())
except Exception as e:
_fail(f"Failed to parse openenv.yaml: {e}")
return False
required_keys = ["name", "version", "description", "tasks", "api",
"observation_space", "action_space", "reward_function"]
missing = [k for k in required_keys if k not in data]
if missing:
_fail(f"Missing required keys: {missing}")
return False
task_ids = [t["id"] for t in data.get("tasks", [])]
for tid in ["easy", "medium", "hard", "cascade"]:
if tid in task_ids:
_ok(f"Task '{tid}' defined")
else:
_fail(f"Task '{tid}' missing from yaml")
return False
_ok("openenv.yaml valid β€” all required fields present")
return True
def check_imports() -> bool:
_header("Check 2 β€” Module imports")
modules = [
"models.action",
"models.observation",
"models.reward",
"app.state",
"app.utils",
"app.actions",
"app.rewards",
"app.graders",
"app.env",
"tasks.task_easy",
"tasks.task_medium",
"tasks.task_hard",
"tasks.task_cascade",
"api.main",
]
ok = True
for mod in modules:
try:
importlib.import_module(mod)
_ok(f"import {mod}")
except Exception as e:
_fail(f"import {mod}: {e}")
ok = False
return ok
def check_tasks_reset() -> bool:
_header("Check 3 β€” reset() for all 4 tasks")
from app.env import ETLDebugEnv
from models.observation import Observation
env = ETLDebugEnv()
ok = True
for task_id in ["easy", "medium", "hard", "cascade"]:
try:
obs = env.reset(task_id=task_id)
assert isinstance(obs, Observation), "reset() must return Observation"
assert obs.tables_preview, "tables_preview must be non-empty"
assert obs.task_description, "task_description must be non-empty"
assert obs.schema_expected, "schema_expected must be non-empty"
assert obs.step_count == 0, "step_count must be 0 after reset"
assert obs.available_actions, "available_actions must be non-empty"
_ok(f"reset('{task_id}') β†’ tables: {list(obs.tables_preview.keys())}")
except Exception as e:
_fail(f"reset('{task_id}'): {e}")
traceback.print_exc()
ok = False
return ok
def check_step() -> bool:
_header("Check 4 β€” step() mechanics")
from app.env import ETLDebugEnv
from models.action import Action
from models.observation import Observation
from models.reward import Reward
env = ETLDebugEnv()
env.reset(task_id="easy")
ok = True
# Valid action
try:
action = Action(action_type="check_nulls", parameters={"table": "users"})
obs, reward, done, info = env.step(action)
assert isinstance(obs, Observation), "step() obs must be Observation"
assert isinstance(reward, Reward), "step() reward must be Reward"
assert isinstance(done, bool), "step() done must be bool"
assert isinstance(info, dict), "step() info must be dict"
assert 0.0 <= reward.total <= 1.0, "reward.total must be in [0, 1]"
assert obs.step_count == 1, "step_count must be 1 after one step"
_ok("Valid action β†’ correct return types, reward in [0,1]")
except Exception as e:
_fail(f"Valid action step: {e}")
ok = False
# Invalid action
try:
action = Action(action_type="check_nulls", parameters={"table": "NONEXISTENT"})
obs, reward, done, info = env.step(action)
assert not done, "should not be done after invalid action"
assert "ERROR" in obs.last_action_result, "invalid action should yield ERROR message"
assert env._state.invalid_action_count >= 1, "invalid_action_count should increment"
_ok("Invalid action β†’ ERROR result, invalid_action_count incremented")
except Exception as e:
_fail(f"Invalid action step: {e}")
ok = False
return ok
def check_new_actions() -> bool:
_header("Check 5 β€” New actions: check_pipeline_health and audit_log")
from app.env import ETLDebugEnv
from models.action import Action
ok = True
for task_id in ["easy", "cascade"]:
env = ETLDebugEnv()
env.reset(task_id=task_id)
# check_pipeline_health
try:
action = Action(action_type="check_pipeline_health", parameters={})
obs, reward, done, info = env.step(action)
assert not done, "should not finish after health check"
assert "PIPELINE HEALTH REPORT" in obs.last_action_result, "health report missing header"
assert info.get("action_valid") is True, "health check should be valid"
_ok(f"Task '{task_id}': check_pipeline_health works")
except Exception as e:
_fail(f"Task '{task_id}': check_pipeline_health: {e}")
ok = False
# audit_log
try:
action = Action(action_type="audit_log", parameters={})
obs, reward, done, info = env.step(action)
assert not done, "should not finish after audit_log"
assert "AUDIT LOG" in obs.last_action_result, "audit log missing header"
_ok(f"Task '{task_id}': audit_log works")
except Exception as e:
_fail(f"Task '{task_id}': audit_log: {e}")
ok = False
return ok
def check_finish_and_grading() -> bool:
_header("Check 6 β€” finish action and grading")
from app.env import ETLDebugEnv
from models.action import Action
ok = True
for task_id in ["easy", "medium", "hard", "cascade"]:
env = ETLDebugEnv()
env.reset(task_id=task_id)
first_table = list(env._state.tables.keys())[0]
env.step(Action(action_type="check_nulls", parameters={"table": first_table}))
finish = Action(action_type="finish", parameters={})
obs, reward, done, info = env.step(finish)
try:
assert done, "done must be True after finish"
assert reward.final_score is not None, "final_score must not be None on finish"
assert 0.0 <= reward.final_score <= 1.0, "final_score must be in [0.0, 1.0]"
_ok(f"Task '{task_id}': finish β†’ final_score={reward.final_score:.4f}")
except AssertionError as e:
_fail(f"Task '{task_id}': {e}")
ok = False
return ok
def check_grader_determinism() -> bool:
_header("Check 7 β€” Grader determinism (run twice)")
from app.env import ETLDebugEnv
from models.action import Action
ok = True
for task_id in ["easy", "medium", "hard", "cascade"]:
scores = []
for _ in range(2):
env = ETLDebugEnv()
env.reset(task_id=task_id)
finish = Action(action_type="finish", parameters={})
_, reward, _, _ = env.step(finish)
scores.append(reward.final_score)
if scores[0] == scores[1]:
_ok(f"Task '{task_id}': deterministic β†’ {scores[0]:.4f} == {scores[1]:.4f}")
else:
_fail(f"Task '{task_id}': non-deterministic! {scores[0]:.4f} β‰  {scores[1]:.4f}")
ok = False
return ok
def check_state_endpoint() -> bool:
_header("Check 8 β€” state() snapshot")
from app.env import ETLDebugEnv
from models.action import Action
env = ETLDebugEnv()
env.reset(task_id="cascade")
try:
snap = env.state()
assert snap, "state() must return non-empty dict"
assert "task_id" in snap, "state() must contain 'task_id'"
assert "tables" in snap, "state() must contain 'tables'"
assert "schema_expected" in snap, "state() must contain 'schema_expected'"
assert "history" in snap, "state() must contain 'history'"
_ok(f"state() returns valid snapshot with keys: {list(snap.keys())}")
return True
except Exception as e:
_fail(f"state(): {e}")
return False
def check_requirements_txt() -> bool:
_header("Check 9 β€” requirements.txt")
path = Path("requirements.txt")
if not path.exists():
_fail("requirements.txt not found")
return False
content = path.read_text().strip()
if not content:
_fail("requirements.txt is empty")
return False
lines = [l for l in content.splitlines() if l.strip() and not l.startswith("#")]
_ok(f"requirements.txt has {len(lines)} package(s)")
return True
def check_dockerfile() -> bool:
_header("Check 10 β€” Dockerfile")
path = Path("Dockerfile")
if not path.exists():
_fail("Dockerfile not found")
return False
content = path.read_text()
for keyword in ["FROM", "EXPOSE", "COPY", "CMD"]:
if keyword not in content:
_fail(f"Dockerfile missing '{keyword}' instruction")
return False
_ok("Dockerfile present with FROM / EXPOSE / COPY / CMD")
return True
def check_inference_script() -> bool:
_header("Check 11 β€” inference.py")
path = Path("inference.py")
if not path.exists():
_fail("inference.py not found in project root")
return False
content = path.read_text()
for symbol in ["API_BASE_URL", "MODEL_NAME", "HF_TOKEN", "def main", "[START]", "[STEP]", "[END]"]:
if symbol not in content:
_fail(f"inference.py missing '{symbol}'")
return False
_ok("inference.py present with required variables, main(), and [START]/[STEP]/[END] logging")
return True
def check_difficulty_progression() -> bool:
_header("Check 12 β€” Task difficulty progression")
from app.env import ETLDebugEnv
from models.action import Action
env = ETLDebugEnv()
scores: Dict[str, float] = {}
for task_id in ["easy", "medium", "hard", "cascade"]:
env.reset(task_id=task_id)
finish = Action(action_type="finish", parameters={})
_, reward, _, _ = env.step(finish)
scores[task_id] = reward.final_score or 0.0
print(f"\n Baseline 'do-nothing' scores:")
for tid, sc in scores.items():
bar = "β–ˆ" * int(sc * 20) + "β–‘" * (20 - int(sc * 20))
print(f" {tid:<10} [{bar}] {sc:.4f}")
# cascade should be the hardest
if scores["cascade"] <= scores["hard"] <= scores["medium"] <= scores["easy"]:
_ok("Difficulty progression confirmed: easy β‰₯ medium β‰₯ hard β‰₯ cascade")
return True
else:
_warn(
f"Difficulty ordering not strict: easy={scores['easy']:.3f} "
f"medium={scores['medium']:.3f} hard={scores['hard']:.3f} "
f"cascade={scores['cascade']:.3f} "
"(acceptable if tasks still differ meaningfully)"
)
return True # warn only, not a hard failure
# ── Main runner ───────────────────────────────────────────────────────────────
def main() -> None:
print(f"\n{BOLD}{'='*60}{RESET}")
print(f"{BOLD} Data Pipeline Incident Response OpenEnv{RESET}")
print(f"{BOLD} Pre-Submission Validator{RESET}")
print(f"{BOLD}{'='*60}{RESET}")
checks: List[Tuple[str, Callable[[], bool]]] = [
("openenv.yaml", check_yaml),
("Module imports", check_imports),
("reset() all 4 tasks", check_tasks_reset),
("step() mechanics", check_step),
("New actions", check_new_actions),
("finish + grading", check_finish_and_grading),
("Grader determinism", check_grader_determinism),
("state() snapshot", check_state_endpoint),
("requirements.txt", check_requirements_txt),
("Dockerfile", check_dockerfile),
("inference.py", check_inference_script),
("Difficulty progression", check_difficulty_progression),
]
results: Dict[str, bool] = {}
for name, fn in checks:
try:
results[name] = fn()
except Exception as exc:
print(f"\n {FAIL} Unexpected error in '{name}': {exc}")
traceback.print_exc()
results[name] = False
print(f"\n{BOLD}{'='*60}{RESET}")
print(f"{BOLD} VALIDATION SUMMARY{RESET}")
print(f"{BOLD}{'='*60}{RESET}")
passed = sum(1 for v in results.values() if v)
total = len(results)
for name, ok in results.items():
status = PASS if ok else FAIL
print(f" {status} {name}")
print(f"\n Result: {passed}/{total} checks passed")
if passed == total:
print(f"\n{GREEN}{BOLD} βœ“ ALL CHECKS PASSED β€” ready to submit!{RESET}\n")
sys.exit(0)
else:
failed = [n for n, v in results.items() if not v]
print(f"\n{RED}{BOLD} βœ— {total - passed} check(s) failed: {failed}{RESET}\n")
sys.exit(1)
if __name__ == "__main__":
main()