| """ |
| Sample test cases for the /evaluate/ endpoint. |
| |
| Sends requests to the deployed evaluator at: |
| https://thinkahead-oe-eval-bcb-lite-evaluator.hf.space/evaluate/ |
| |
| Each test case constructs a payload with: |
| - task_id, res_id, solution, test (a unittest.TestCase), entry_point, code_prompt |
| and validates the response status ("pass", "fail", or "timeout"). |
| """ |
|
|
| import httpx |
| import json |
| import sys |
|
|
| BASE_URL = "https://thinkahead-oe-eval-bcb-lite-evaluator.hf.space" |
| EVALUATE_URL = f"{BASE_URL}/evaluate/" |
| TIMEOUT = 300 |
|
|
|
|
| def make_sample(task_id, res_id, code_prompt, solution, test, entry_point): |
| return { |
| "task_id": task_id, |
| "res_id": res_id, |
| "code_prompt": code_prompt, |
| "solution": solution, |
| "test": test, |
| "entry_point": entry_point, |
| } |
|
|
|
|
| |
| |
| |
|
|
| |
| CASE_PASS_ADD = make_sample( |
| task_id="test/add_pass", |
| res_id=0, |
| code_prompt="def add(a, b):", |
| solution="def add(a, b):\n return a + b\n", |
| test=""" |
| import unittest |
| class TestCases(unittest.TestCase): |
| def test_add_positive(self): |
| self.assertEqual(add(2, 3), 5) |
| def test_add_negative(self): |
| self.assertEqual(add(-1, -1), -2) |
| def test_add_zero(self): |
| self.assertEqual(add(0, 0), 0) |
| """, |
| entry_point="add", |
| ) |
|
|
| |
| CASE_FAIL_MULTIPLY = make_sample( |
| task_id="test/multiply_fail", |
| res_id=1, |
| code_prompt="def multiply(a, b):", |
| solution="def multiply(a, b):\n return a + b\n", |
| test=""" |
| import unittest |
| class TestCases(unittest.TestCase): |
| def test_multiply_basic(self): |
| self.assertEqual(multiply(3, 4), 12) |
| def test_multiply_zero(self): |
| self.assertEqual(multiply(0, 5), 0) |
| """, |
| entry_point="multiply", |
| ) |
|
|
| |
| CASE_PASS_FIB = make_sample( |
| task_id="test/fibonacci_pass", |
| res_id=2, |
| code_prompt="def fibonacci(n):", |
| solution="""def fibonacci(n): |
| if n <= 0: |
| return 0 |
| elif n == 1: |
| return 1 |
| a, b = 0, 1 |
| for _ in range(2, n + 1): |
| a, b = b, a + b |
| return b |
| """, |
| test=""" |
| import unittest |
| class TestCases(unittest.TestCase): |
| def test_fib_zero(self): |
| self.assertEqual(fibonacci(0), 0) |
| def test_fib_one(self): |
| self.assertEqual(fibonacci(1), 1) |
| def test_fib_ten(self): |
| self.assertEqual(fibonacci(10), 55) |
| def test_fib_twenty(self): |
| self.assertEqual(fibonacci(20), 6765) |
| """, |
| entry_point="fibonacci", |
| ) |
|
|
| |
| CASE_FAIL_EXCEPTION = make_sample( |
| task_id="test/divide_exception", |
| res_id=3, |
| code_prompt="def safe_divide(a, b):", |
| solution="""def safe_divide(a, b): |
| return a / b |
| """, |
| test=""" |
| import unittest |
| class TestCases(unittest.TestCase): |
| def test_divide_normal(self): |
| self.assertEqual(safe_divide(10, 2), 5.0) |
| def test_divide_by_zero(self): |
| self.assertEqual(safe_divide(1, 0), 0) |
| """, |
| entry_point="safe_divide", |
| ) |
|
|
| |
| CASE_PASS_REVERSE = make_sample( |
| task_id="test/reverse_pass", |
| res_id=4, |
| code_prompt="def reverse_string(s):", |
| solution="def reverse_string(s):\n return s[::-1]\n", |
| test=""" |
| import unittest |
| class TestCases(unittest.TestCase): |
| def test_reverse_word(self): |
| self.assertEqual(reverse_string("hello"), "olleh") |
| def test_reverse_empty(self): |
| self.assertEqual(reverse_string(""), "") |
| def test_reverse_palindrome(self): |
| self.assertEqual(reverse_string("racecar"), "racecar") |
| """, |
| entry_point="reverse_string", |
| ) |
|
|
| |
| CASE_PASS_FLATTEN = make_sample( |
| task_id="test/flatten_pass", |
| res_id=5, |
| code_prompt="def flatten(lst):", |
| solution="""def flatten(lst): |
| result = [] |
| for item in lst: |
| if isinstance(item, list): |
| result.extend(flatten(item)) |
| else: |
| result.append(item) |
| return result |
| """, |
| test=""" |
| import unittest |
| class TestCases(unittest.TestCase): |
| def test_flat(self): |
| self.assertEqual(flatten([1, 2, 3]), [1, 2, 3]) |
| def test_nested(self): |
| self.assertEqual(flatten([1, [2, 3], [4, [5]]]), [1, 2, 3, 4, 5]) |
| def test_empty(self): |
| self.assertEqual(flatten([]), []) |
| """, |
| entry_point="flatten", |
| ) |
|
|
|
|
| ALL_CASES = [ |
| ("PASS add", CASE_PASS_ADD, "pass"), |
| ("FAIL multiply", CASE_FAIL_MULTIPLY, "fail"), |
| ("PASS fibonacci", CASE_PASS_FIB, "pass"), |
| ("FAIL divide_by_zero",CASE_FAIL_EXCEPTION, "fail"), |
| ("PASS reverse_string",CASE_PASS_REVERSE, "pass"), |
| ("PASS flatten", CASE_PASS_FLATTEN, "pass"), |
| ] |
|
|
|
|
| def run_tests(): |
| print(f"Sending {len(ALL_CASES)} samples to {EVALUATE_URL}") |
| print(f"(calibrate=False so solutions are evaluated as-is)\n") |
|
|
| samples = [case for _, case, _ in ALL_CASES] |
| expected = {case["task_id"]: exp for _, case, exp in ALL_CASES} |
|
|
| |
| with httpx.Client(timeout=TIMEOUT) as client: |
| resp = client.post( |
| EVALUATE_URL, |
| json=samples, |
| params={"calibrate": False, "min_time_limit": 1}, |
| ) |
|
|
| if resp.status_code != 200: |
| print(f"ERROR: HTTP {resp.status_code}") |
| print(resp.text) |
| sys.exit(1) |
|
|
| results = resp.json() |
|
|
| print(f"Date: {results['date']}\n") |
| print(f"{'Test Case':<25} {'Expected':<10} {'Actual':<10} {'Result'}") |
| print("-" * 60) |
|
|
| passed = 0 |
| total = 0 |
| for label, case, exp_status in ALL_CASES: |
| task_id = case["task_id"] |
| task_results = results["eval"].get(task_id, []) |
| if not task_results: |
| actual = "MISSING" |
| ok = False |
| else: |
| actual = task_results[0]["status"] |
| ok = actual == exp_status |
|
|
| status_str = "OK" if ok else "MISMATCH" |
| print(f"{label:<25} {exp_status:<10} {actual:<10} {status_str}") |
|
|
| if not ok and task_results: |
| details = task_results[0].get("details", {}) |
| if details: |
| for k, v in details.items(): |
| |
| first_line = str(v).split("\n")[0][:80] |
| print(f" detail[{k}]: {first_line}") |
|
|
| total += 1 |
| if ok: |
| passed += 1 |
|
|
| print("-" * 60) |
| print(f"\n{passed}/{total} test cases matched expected status.") |
|
|
| if passed == total: |
| print("\nAll tests produced expected results.") |
| else: |
| print("\nSome tests did not match. Review details above.") |
| sys.exit(1) |
|
|
|
|
| if __name__ == "__main__": |
| run_tests() |
|
|
