File size: 3,374 Bytes
3b0d6b3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | import enum
from datetime import datetime
from sqlalchemy import Column, DateTime, Enum, Float, ForeignKey, Integer, String, Text
from sqlalchemy.orm import relationship
from database import Base
class RunStatus(enum.Enum):
pending = "pending"
running = "running"
success = "success"
failure = "failure"
error = "error"
timeout = "timeout"
class Task(Base):
__tablename__ = "tasks"
id = Column(Integer, primary_key=True, index=True)
slug = Column(String(200), unique=True, nullable=False, index=True)
title = Column(String(500), nullable=False)
description = Column(Text)
repo_url = Column(String(500), nullable=False)
repo_ref = Column(String(200), default="main")
success_cmd = Column(String(1000), nullable=False)
time_limit_s = Column(Integer, default=300)
token_limit = Column(Integer, default=200000)
difficulty = Column(String(50), default="medium")
created_at = Column(DateTime, default=datetime.utcnow)
runs = relationship("Run", back_populates="task", cascade="all, delete-orphan")
class Agent(Base):
__tablename__ = "agents"
id = Column(Integer, primary_key=True, index=True)
name = Column(String(200), nullable=False, unique=True)
adapter_type = Column(String(100), nullable=False) # "claude_code", "manual"
config_json = Column(Text, default="{}")
created_at = Column(DateTime, default=datetime.utcnow)
runs = relationship("Run", back_populates="agent", cascade="all, delete-orphan")
class Run(Base):
__tablename__ = "runs"
id = Column(Integer, primary_key=True, index=True)
task_id = Column(Integer, ForeignKey("tasks.id"), nullable=False)
agent_id = Column(Integer, ForeignKey("agents.id"), nullable=False)
status = Column(Enum(RunStatus), default=RunStatus.pending)
duration_s = Column(Float, default=0.0)
tokens_input = Column(Integer, default=0)
tokens_output = Column(Integer, default=0)
cost_usd = Column(Float, default=0.0)
tests_total = Column(Integer, default=0)
tests_passed = Column(Integer, default=0)
errors_count = Column(Integer, default=0)
tool_calls = Column(Integer, default=0)
diff_text = Column(Text, default="")
trajectory = Column(Text, default="[]") # JSON array
judge_score = Column(Float)
judge_notes = Column(Text)
pass_at_k_group = Column(String(100)) # groups runs from same pass@k batch
metrics_json = Column(Text, default="{}") # JSON: full evaluation metrics
functional_success = Column(Integer, default=0)
test_pass_rate = Column(Float)
alignment_score_llm = Column(Float)
alignment_score_human = Column(Float)
alignment_rubric_version = Column(String(100))
quality_score = Column(Float)
agent_duration_s = Column(Float, default=0.0)
end_to_end_duration_s = Column(Float, default=0.0)
timeout_flag = Column(Integer, default=0)
tool_failure_flag = Column(Integer, default=0)
invalid_patch_flag = Column(Integer, default=0)
task_family = Column(String(100))
seed = Column(String(100))
replicate_index = Column(Integer, default=0)
experiment_id = Column(String(100))
arm_id = Column(String(100))
created_at = Column(DateTime, default=datetime.utcnow)
task = relationship("Task", back_populates="runs")
agent = relationship("Agent", back_populates="runs")
|