File size: 4,648 Bytes
d02897f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """
Pre-submission validation - Run this before deploying!
"""
import os
import sys
import subprocess
def check(condition, message):
"""Helper to check condition."""
if condition:
print(f"โ
{message}")
return True
else:
print(f"โ {message}")
return False
def validate():
"""Run all validation checks."""
print("="*60)
print("OpenOps Pre-Submission Validation")
print("="*60)
print()
checks = []
# File existence
print("๐ Checking required files...")
checks.append(check(os.path.exists("models.py"), "models.py exists"))
checks.append(check(os.path.exists("server/my_env_environment.py"), "server/my_env_environment.py exists"))
checks.append(check(os.path.exists("graders.py"), "graders.py exists"))
checks.append(check(os.path.exists("inference.py"), "inference.py exists"))
checks.append(check(os.path.exists("openenv.yaml"), "openenv.yaml exists"))
checks.append(check(os.path.exists("README.md"), "README.md exists"))
checks.append(check(os.path.exists("server/Dockerfile"), "server/Dockerfile exists"))
checks.append(check(os.path.exists("server/requirements.txt"), "server/requirements.txt exists"))
checks.append(check(os.path.exists("server/app.py"), "server/app.py exists"))
checks.append(check(os.path.exists("client.py"), "client.py exists"))
print()
# Import test
print("๐ง Testing imports...")
try:
from models import IncidentAction, IncidentObservation, IncidentState
from server.my_env_environment import MyEnvEnvironment
from graders import get_grader
checks.append(check(True, "All imports successful"))
except Exception as e:
checks.append(check(False, f"Import failed: {e}"))
print()
# Environment test
print("๐ฎ Testing environment...")
try:
from server.my_env_environment import MyEnvEnvironment
from models import IncidentAction
env = MyEnvEnvironment()
obs = env.reset(task_id=1)
checks.append(check(obs is not None, "Environment resets"))
action = IncidentAction(action_id=0, task_id=1)
obs = env.step(action)
checks.append(check(obs.reward is not None, "Environment steps"))
except Exception as e:
checks.append(check(False, f"Environment test failed: {e}"))
print()
# Grader test
print("๐ Testing graders...")
try:
from graders import get_grader
from server.my_env_environment import MyEnvEnvironment
env = MyEnvEnvironment()
env.reset(task_id=1)
grader = get_grader(1)
score = grader(env)
checks.append(check(0.0 <= score <= 1.0, f"Grader works (score: {score:.2f})"))
except Exception as e:
checks.append(check(False, f"Grader test failed: {e}"))
print()
# Inference script test
print("๐ค Testing inference script...")
try:
with open("inference.py", "r") as f:
content = f.read()
has_start = "[START]" in content
has_step = "[STEP]" in content
has_end = "[END]" in content
checks.append(check(has_start and has_step and has_end, "Has required log format"))
checks.append(check("from openai import OpenAI" in content or "OpenAI" in content, "Uses OpenAI-compatible client"))
except Exception as e:
checks.append(check(False, f"Inference validation failed: {e}"))
print()
# README check
print("๐ Checking README...")
try:
with open("README.md", "r") as f:
readme = f.read()
checks.append(check(len(readme) > 500, "README has content (>500 chars)"))
checks.append(check("OpenOps" in readme, "README mentions project name"))
checks.append(check("Task 1" in readme or "Task 2" in readme, "README describes tasks"))
except Exception as e:
checks.append(check(False, f"README check failed: {e}"))
print()
# Summary
print("="*60)
passed = sum(checks)
total = len(checks)
print(f"\nResults: {passed}/{total} checks passed")
if passed == total:
print("\nโ
ALL CHECKS PASSED - READY TO SUBMIT! ๐")
return True
else:
print(f"\nโ ๏ธ {total - passed} issues found - FIX BEFORE SUBMITTING!")
return False
if __name__ == "__main__":
success = validate()
sys.exit(0 if success else 1) |