File size: 2,488 Bytes
ad2fa65 2f2ad55 ad2fa65 9f0f6f0 c1aaf22 ad2fa65 9f0f6f0 ad2fa65 2f2ad55 c1aaf22 ad2fa65 2f2ad55 ad2fa65 9d81799 ad2fa65 9f0f6f0 ad2fa65 2f2ad55 ad2fa65 2f2ad55 ad2fa65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | """Run the assignment agent locally without submitting results."""
import argparse
import textwrap
import requests
from agent import BasicAgent
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--limit",
type=int,
default=None,
help="Evaluate only the first N questions.",
)
parser.add_argument(
"--start",
type=int,
default=0,
help="Skip the first N questions.",
)
parser.add_argument(
"--task-id",
action="append",
dest="task_ids",
help="Evaluate only these task ids. Repeatable.",
)
parser.add_argument(
"--model",
default=None,
help="Model id to use, e.g. anthropic/claude-opus-4-1. Defaults to the configured model.",
)
args = parser.parse_args()
response = requests.get(f"{DEFAULT_API_URL}/questions", timeout=15)
response.raise_for_status()
questions = response.json()
if args.task_ids:
questions = [q for q in questions if q.get("task_id") in set(args.task_ids)]
else:
questions = questions[args.start:]
if args.limit is not None:
questions = questions[:args.limit]
print("\n" + "=" * 72)
print(f"LOCAL EVALUATION | {len(questions)} question(s)")
print("Nothing will be submitted.")
print("=" * 72)
agent = BasicAgent(verbose=True, model_id=args.model)
for index, item in enumerate(questions, start=1):
task_id = item.get("task_id")
question = item.get("question")
if not task_id or question is None:
print(f"\n--- Question {index} | SKIPPED ---")
print(f"Malformed question: {item}")
continue
print(f"\n--- Question {index} | {task_id} | TRACE ---")
print("Question:")
print(textwrap.indent(question, " "))
print("Loop:")
try:
answer = agent(question, task_id=task_id, file_name=item.get("file_name"))
except Exception as error:
print(f"\n--- Question {index} | {task_id} | ERROR ---")
print(f"{error}")
continue
print(f"\n--- Question {index} | {task_id} ---")
print("Answer:")
print(textwrap.indent(answer, " "))
print("\n" + "=" * 72)
print("LOCAL EVALUATION COMPLETE")
print("=" * 72)
if __name__ == "__main__":
main()
|