Upload folder using huggingface_hub

08e5967 verified 7 months ago

19.3 kB

#!/usr/bin/env python3
"""
AgentBench OS Evaluation Harness
Runs tool-use tasks against LLM backends via Docker containers.
Uses prompt-based tool calling (Qwen3.5 native format) since
none of the backends support API-level tools for this model.
Usage:
  python3 run_bench.py --backend ollama --dataset agentbench --version v1
  python3 run_bench.py --backend mlx-vlm --dataset agentbench --version v2
  python3 run_bench.py --backend llama-cpp --dataset agentbench --version v1
"""

import argparse
import json
import os
import re
import subprocess
import sys
import time
import requests

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
REPO_DIR = os.path.dirname(SCRIPT_DIR)
BENCHMARKS_DIR = os.environ.get('BENCHMARKS_DIR', REPO_DIR)
SCRIPTS_DIR = os.environ.get('SCRIPTS_DIR', os.path.join(BENCHMARKS_DIR, 'agentbench_scripts'))
DOCKER_HOST = os.environ.get('DOCKER_HOST', 'unix:///var/run/docker.sock')
MAX_ROUNDS = 8
BASH_OUTPUT_LIMIT = 800

BACKEND_CONFIG = {
    "ollama": {
        "url": "http://localhost:11434/api/chat",
        "model": "qwen3.5:9b",
        "native": True,  # Use Ollama native API format
    },
    "mlx-vlm": {
        "url": "http://localhost:8080/chat/completions",
        "model": "mlx-community/Qwen3.5-9B-MLX-4bit",
    },
    "llama-cpp": {
        "url": "http://localhost:8081/v1/chat/completions",
        "model": "qwen3.5-9b",
    },
}

SYSTEM_PROMPT = """You are an AI assistant with access to bash. You solve tasks by running commands.
Available actions (output as JSON):
1. Run bash: {"action": "bash", "script": "your command here"}
2. Submit answer: {"action": "answer", "value": "your answer"}
Output ONLY one JSON action per turn. No explanation needed.
For numeric answers, output just the number.
You have 8 turns maximum. Be efficient."""

# Alternate tool call patterns to parse
TOOL_PATTERNS = [
    # JSON action format
    r'\{"action":\s*"bash",\s*"script":\s*"(.*?)"\}',
    r'\{"action":\s*"answer",\s*"value":\s*"(.*?)"\}',
    # tool_call format (Qwen native)
    r'<tool_call>\s*\{.*?\}\s*</tool_call>',
]


def docker_exec(container_id, cmd, timeout=30):
    """Execute a command in a Docker container and return stdout."""
    env = os.environ.copy()
    env["DOCKER_HOST"] = DOCKER_HOST
    try:
        result = subprocess.run(
            ["docker", "exec", container_id, "bash", "-c", cmd],
            capture_output=True,
            text=True,
            timeout=timeout,
            env=env,
        )
        output = result.stdout + result.stderr
        if len(output) > BASH_OUTPUT_LIMIT:
            output = output[:BASH_OUTPUT_LIMIT] + "\n... (truncated)"
        return output.strip()
    except subprocess.TimeoutExpired:
        return "(command timed out after {}s)".format(timeout)
    except Exception as e:
        return f"(error: {e})"


def start_container(task):
    """Start a Docker container for a task, run init and start commands."""
    env = os.environ.copy()
    env["DOCKER_HOST"] = DOCKER_HOST

    create = task.get("create", {})
    if isinstance(create, dict):
        image_name = create.get("local", "default")
    else:
        image_name = "default"

    docker_image = f"agentbench-{image_name}"

    result = subprocess.run(
        ["docker", "run", "-d", "--rm", docker_image, "sleep", "3600"],
        capture_output=True,
        text=True,
        env=env,
    )
    container_id = result.stdout.strip()
    if not container_id:
        print(f"  ERROR: Failed to start container: {result.stderr}")
        return None

    print(f"  Container: {container_id[:12]} (image: {docker_image})")

    if isinstance(create, dict) and "init" in create:
        init = create["init"]
        if "code" in init:
            docker_exec(container_id, init["code"])
        elif "file" in init:
            init_file = os.path.join(SCRIPTS_DIR, init["file"])
            if os.path.exists(init_file):
                subprocess.run(
                    ["docker", "cp", init_file, f"{container_id}:/tmp/init.sh"],
                    env=env,
                )
                docker_exec(container_id, "bash /tmp/init.sh")

    start_cmd = task.get("start")
    if start_cmd:
        docker_exec(container_id, start_cmd, timeout=5)
        time.sleep(1)

    return container_id


def stop_container(container_id):
    """Stop a Docker container."""
    env = os.environ.copy()
    env["DOCKER_HOST"] = DOCKER_HOST
    subprocess.run(["docker", "kill", container_id], capture_output=True, env=env)


def call_llm(backend, messages):
    """Call the LLM backend and return the response.
    Handles both OpenAI-compatible and Ollama native API formats.
    """
    config = BACKEND_CONFIG[backend]
    is_native = config.get("native", False)

    if is_native:
        # Ollama native API format
        payload = {
            "model": config["model"],
            "messages": messages,
            "stream": False,
            "think": False,
            "options": {"temperature": 0, "num_predict": 2048},
        }
    else:
        # OpenAI-compatible format
        payload = {
            "model": config["model"],
            "messages": messages,
            "temperature": 0,
            "max_tokens": 2048,
        }

    try:
        resp = requests.post(config["url"], json=payload, timeout=180)
        resp.raise_for_status()
        data = resp.json()

        if is_native:
            # Convert Ollama native response to OpenAI-compatible format
            msg = data.get("message", {})
            content = msg.get("content", "")
            prompt_tokens = data.get("prompt_eval_count", 0)
            eval_tokens = data.get("eval_count", 0)
            return {
                "choices": [{"message": {"role": "assistant", "content": content}}],
                "usage": {
                    "prompt_tokens": prompt_tokens,
                    "completion_tokens": eval_tokens,
                    "total_tokens": prompt_tokens + eval_tokens,
                },
            }
        return data
    except requests.exceptions.ConnectionError:
        return {"error": f"Cannot connect to {backend} at {config['url']}"}
    except Exception as e:
        return {"error": str(e)}


def parse_tool_calls(text):
    """Parse tool calls from the model's text output.
    Handles multiple formats:
    1. JSON action format: {"action": "bash", "script": "ls"}
    2. Qwen tool_call tags: <tool_call>{"name": "bash_action", ...}</tool_call>
    3. Raw JSON with name/arguments
    """
    # Strip thinking content (handles both closed and unclosed think tags)
    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
    text = re.sub(r"<think>.*", "", text, flags=re.DOTALL)  # unclosed
    text = re.sub(r"</think>", "", text)  # orphaned closing
    text = re.sub(r"Thinking Process:.*?(?=\{|<tool_call>|$)", "", text, flags=re.DOTALL)

    tool_calls = []

    # Format 1: Find JSON objects (handles nested braces via balanced extraction)
    for start_idx in [m.start() for m in re.finditer(r'\{', text)]:
        depth = 0
        end_idx = start_idx
        in_string = False
        escape = False
        for i in range(start_idx, len(text)):
            c = text[i]
            if escape:
                escape = False
                continue
            if c == '\\':
                escape = True
                continue
            if c == '"' and not escape:
                in_string = not in_string
                continue
            if in_string:
                continue
            if c == '{':
                depth += 1
            elif c == '}':
                depth -= 1
                if depth == 0:
                    end_idx = i + 1
                    break
        if depth == 0 and end_idx > start_idx:
            candidate = text[start_idx:end_idx]
            try:
                obj = json.loads(candidate)
                action = obj.get("action", "")
                if action == "bash" and "script" in obj:
                    tool_calls.append({"name": "bash_action", "arguments": {"script": obj["script"]}})
                elif action == "answer" and "value" in obj:
                    tool_calls.append({"name": "answer_action", "arguments": {"answer": str(obj["value"])}})
                elif obj.get("name") in ("bash_action", "answer_action"):
                    tool_calls.append(obj)
                if tool_calls:
                    break  # Take the first valid tool call
            except (json.JSONDecodeError, ValueError):
                pass

    if tool_calls:
        return tool_calls

    # Format 2: <tool_call>...</tool_call> blocks
    matches = re.findall(r"<tool_call>\s*(.*?)\s*</tool_call>", text, re.DOTALL)
    for m in matches:
        try:
            tc = json.loads(m.strip())
            tool_calls.append(tc)
        except json.JSONDecodeError:
            json_match = re.search(r"\{.*\}", m, re.DOTALL)
            if json_match:
                try:
                    tc = json.loads(json_match.group())
                    tool_calls.append(tc)
                except json.JSONDecodeError:
                    pass

    return tool_calls


def evaluate_task(task, answer, container_id):
    """Evaluate the agent's answer against the task's oracle."""
    evaluation = task.get("evaluation", {})

    if "match" in evaluation:
        expected = evaluation["match"]
        if isinstance(expected, dict):
            if "answer" in expected:
                return answer.strip() == expected["answer"].strip()
            if "regex" in expected:
                return bool(re.search(expected["regex"], answer))
        else:
            return answer.strip() == str(expected).strip()

    if "check" in evaluation and "example" in evaluation:
        example = evaluation["example"]
        if isinstance(example, dict):
            example_code = example.get("code", "")
        else:
            example_code = str(example)
        if example_code and container_id:
            ground_truth = docker_exec(container_id, example_code, timeout=15)
            print(f"    Ground truth: {ground_truth[:100]}")

            checks = evaluation["check"]
            for check in checks:
                if check is None:
                    continue
                if isinstance(check, dict):
                    lang = check.get("language", "python")
                    check_file = check.get("file", "")
                    check_path = os.path.join(SCRIPTS_DIR, check_file)

                    if os.path.exists(check_path):
                        try:
                            if lang == "python":
                                result = subprocess.run(
                                    ["python3", check_path, answer.strip(), ground_truth.strip()],
                                    capture_output=True,
                                    timeout=10,
                                )
                            else:
                                result = subprocess.run(
                                    ["bash", check_path, answer.strip(), ground_truth.strip()],
                                    capture_output=True,
                                    timeout=10,
                                )
                            return result.returncode == 0
                        except Exception as e:
                            print(f"    Check script error: {e}")
                            return False

    print(f"    WARNING: Could not evaluate")
    return False


def run_task(task, task_idx, backend):
    """Run a single AgentBench OS task."""
    description = task["description"]
    print(f"\n  Task {task_idx}: {description[:80]}...")

    container_id = start_container(task)
    if not container_id:
        return {"task_idx": task_idx, "passed": False, "error": "container_start_failed", "rounds": 0, "latency": 0}

    messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": description},
    ]

    answer = None
    total_tokens = 0
    start_time = time.time()

    try:
        for round_num in range(MAX_ROUNDS):
            print(f"    Round {round_num + 1}/{MAX_ROUNDS}...", end=" ", flush=True)

            response = call_llm(backend, messages)
            if "error" in response:
                print(f"LLM error: {response['error']}")
                break

            usage = response.get("usage", {})
            total_tokens += usage.get("total_tokens", 0)

            choices = response.get("choices", [])
            if not choices:
                print("no choices")
                break

            content = choices[0].get("message", {}).get("content", "")
            if not content:
                print("empty response")
                break

            # Parse tool calls from the text
            tool_calls = parse_tool_calls(content)

            if not tool_calls:
                # No tool calls found — maybe the model answered directly
                clean = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
                clean = re.sub(r"<think>.*", "", clean, flags=re.DOTALL).strip()
                clean = re.sub(r"</think>", "", clean).strip()
                clean = re.sub(r"Thinking Process:.*", "", clean, flags=re.DOTALL).strip()
                if clean:
                    # Try to extract just a number or short answer from verbose response
                    lines = clean.strip().split("\n")
                    last_line = lines[-1].strip()
                    # If last line is a bare number, use it
                    if re.match(r"^-?\d+\.?\d*$", last_line):
                        answer = last_line
                    else:
                        answer = clean
                    print(f"direct answer: {answer[:80]}")
                else:
                    print("no tool calls and no useful content")
                messages.append({"role": "assistant", "content": content})
                break

            messages.append({"role": "assistant", "content": content})

            # Process tool calls
            tool_results = []
            for tc in tool_calls:
                name = tc.get("name", "")
                args = tc.get("arguments", {})

                if name == "answer_action":
                    answer = str(args.get("answer", ""))
                    print(f"ANSWER: {answer}")
                    tool_results.append(f"Answer recorded: {answer}")
                    break

                elif name == "bash_action":
                    script = args.get("script", "")
                    output = docker_exec(container_id, script)
                    short_script = script[:50].replace('\n', ' ')
                    short_output = output[:50].replace('\n', ' ')
                    print(f"bash({short_script}) -> {short_output}")
                    tool_results.append(output if output else "(no output)")
                else:
                    print(f"unknown: {name}")
                    tool_results.append(f"Unknown tool: {name}")

            # Feed tool results back
            if tool_results and answer is None:
                result_text = "\n".join(f"<tool_response>\n{r}\n</tool_response>" for r in tool_results)
                messages.append({"role": "user", "content": result_text})

            if answer is not None:
                break

        elapsed = time.time() - start_time

        if answer is not None:
            passed = evaluate_task(task, answer, container_id)
            print(f"    Result: {'PASS' if passed else 'FAIL'} (answer={answer}, time={elapsed:.1f}s)")
        else:
            passed = False
            print(f"    Result: FAIL (no answer, time={elapsed:.1f}s)")

        return {
            "task_idx": task_idx,
            "description": description[:100],
            "passed": passed,
            "answer": answer,
            "rounds": round_num + 1,
            "latency": round(elapsed, 2),
            "total_tokens": total_tokens,
        }

    finally:
        stop_container(container_id)


def main():
    parser = argparse.ArgumentParser(description="AgentBench OS Evaluation Harness")
    parser.add_argument("--backend", choices=["ollama", "mlx-vlm", "llama-cpp"], required=True)
    parser.add_argument("--dataset", choices=["agentbench"], default="agentbench")
    parser.add_argument("--version", choices=["v1", "v2", "v3", "v4"], default="v1")
    parser.add_argument("--tasks", type=str, default=None, help="Comma-separated task indices")
    parser.add_argument("--model", type=str, default=None, help="Override model name (e.g. llama3.1:8b)")
    args = parser.parse_args()

    sample_file = os.path.join(BENCHMARKS_DIR, f"agentbench_os_{args.version}.json")
    with open(sample_file) as f:
        tasks = json.load(f)

    print(f"=== AgentBench OS {args.version} ===")
    print(f"Backend: {args.backend}")
    print(f"Tasks: {len(tasks)}")
    print(f"Endpoint: {BACKEND_CONFIG[args.backend]['url']}")

    # Check backend
    config = BACKEND_CONFIG[args.backend].copy()
    if args.model:
        config["model"] = args.model
        BACKEND_CONFIG[args.backend] = config
    try:
        if config.get("native"):
            resp = requests.get(config["url"].replace("/api/chat", "/api/tags"), timeout=5)
        else:
            base = config["url"].rsplit("/chat", 1)[0]
            resp = requests.get(base + "/models", timeout=5)
        print(f"Backend reachable: OK ({resp.status_code})")
    except Exception as e:
        print(f"WARNING: Backend check failed: {e}")

    if args.tasks:
        indices = [int(x) for x in args.tasks.split(",")]
        tasks_to_run = [(i, tasks[i]) for i in indices if i < len(tasks)]
    else:
        tasks_to_run = list(enumerate(tasks))

    results = []
    for task_idx, task in tasks_to_run:
        result = run_task(task, task_idx, args.backend)
        results.append(result)

    passed = sum(1 for r in results if r["passed"])
    total = len(results)
    avg_latency = sum(r["latency"] for r in results) / total if total else 0
    avg_tokens = sum(r["total_tokens"] for r in results) / total if total else 0

    print(f"\n{'='*50}")
    print(f"RESULTS: {args.backend} / agentbench_os_{args.version}")
    print(f"Passed: {passed}/{total}")
    print(f"Avg latency: {avg_latency:.1f}s")
    print(f"Avg tokens: {avg_tokens:.0f}")
    print(f"{'='*50}")

    model_tag = config["model"].replace("/", "_").replace(":", "_")
    results_dir = os.path.join(BENCHMARKS_DIR, "results", model_tag)
    os.makedirs(results_dir, exist_ok=True)
    results_file = os.path.join(results_dir, f"agentbench_os_{args.version}.json")
    print(f"Model: {config['model']}")
    with open(results_file, "w") as f:
        json.dump({
            "backend": args.backend,
            "dataset": "agentbench_os",
            "version": args.version,
            "model": config["model"],
            "passed": passed,
            "total": total,
            "avg_latency": round(avg_latency, 2),
            "avg_tokens": round(avg_tokens),
            "tasks": results,
        }, f, indent=2)

    print(f"Results saved to: {results_file}")
    return 0 if passed == total else 1


if __name__ == "__main__":
    sys.exit(main())

	#!/usr/bin/env python3
	"""
	AgentBench OS Evaluation Harness
	Runs tool-use tasks against LLM backends via Docker containers.
	Uses prompt-based tool calling (Qwen3.5 native format) since
	none of the backends support API-level tools for this model.

	Usage:
	python3 run_bench.py --backend ollama --dataset agentbench --version v1
	python3 run_bench.py --backend mlx-vlm --dataset agentbench --version v2
	python3 run_bench.py --backend llama-cpp --dataset agentbench --version v1
	"""

	import argparse
	import json
	import os
	import re
	import subprocess
	import sys
	import time
	import requests

	SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
	REPO_DIR = os.path.dirname(SCRIPT_DIR)
	BENCHMARKS_DIR = os.environ.get('BENCHMARKS_DIR', REPO_DIR)
	SCRIPTS_DIR = os.environ.get('SCRIPTS_DIR', os.path.join(BENCHMARKS_DIR, 'agentbench_scripts'))
	DOCKER_HOST = os.environ.get('DOCKER_HOST', 'unix:///var/run/docker.sock')
	MAX_ROUNDS = 8
	BASH_OUTPUT_LIMIT = 800

	BACKEND_CONFIG = {
	"ollama": {
	"url": "http://localhost:11434/api/chat",
	"model": "qwen3.5:9b",
	"native": True, # Use Ollama native API format
	},
	"mlx-vlm": {
	"url": "http://localhost:8080/chat/completions",
	"model": "mlx-community/Qwen3.5-9B-MLX-4bit",
	},
	"llama-cpp": {
	"url": "http://localhost:8081/v1/chat/completions",
	"model": "qwen3.5-9b",
	},
	}

	SYSTEM_PROMPT = """You are an AI assistant with access to bash. You solve tasks by running commands.

	Available actions (output as JSON):
	1. Run bash: {"action": "bash", "script": "your command here"}
	2. Submit answer: {"action": "answer", "value": "your answer"}

	Output ONLY one JSON action per turn. No explanation needed.
	For numeric answers, output just the number.
	You have 8 turns maximum. Be efficient."""

	# Alternate tool call patterns to parse
	TOOL_PATTERNS = [
	# JSON action format
	r'\{"action":\s"bash",\s"script":\s"(.?)"\}',
	r'\{"action":\s"answer",\s"value":\s"(.?)"\}',
	# tool_call format (Qwen native)
	r'<tool_call>\s\{.?\}\s*</tool_call>',
	]


	def docker_exec(container_id, cmd, timeout=30):
	"""Execute a command in a Docker container and return stdout."""
	env = os.environ.copy()
	env["DOCKER_HOST"] = DOCKER_HOST
	try:
	result = subprocess.run(
	["docker", "exec", container_id, "bash", "-c", cmd],
	capture_output=True,
	text=True,
	timeout=timeout,
	env=env,
	)
	output = result.stdout + result.stderr
	if len(output) > BASH_OUTPUT_LIMIT:
	output = output[:BASH_OUTPUT_LIMIT] + "\n... (truncated)"
	return output.strip()
	except subprocess.TimeoutExpired:
	return "(command timed out after {}s)".format(timeout)
	except Exception as e:
	return f"(error: {e})"


	def start_container(task):
	"""Start a Docker container for a task, run init and start commands."""
	env = os.environ.copy()
	env["DOCKER_HOST"] = DOCKER_HOST

	create = task.get("create", {})
	if isinstance(create, dict):
	image_name = create.get("local", "default")
	else:
	image_name = "default"

	docker_image = f"agentbench-{image_name}"

	result = subprocess.run(
	["docker", "run", "-d", "--rm", docker_image, "sleep", "3600"],
	capture_output=True,
	text=True,
	env=env,
	)
	container_id = result.stdout.strip()
	if not container_id:
	print(f" ERROR: Failed to start container: {result.stderr}")
	return None

	print(f" Container: {container_id[:12]} (image: {docker_image})")

	if isinstance(create, dict) and "init" in create:
	init = create["init"]
	if "code" in init:
	docker_exec(container_id, init["code"])
	elif "file" in init:
	init_file = os.path.join(SCRIPTS_DIR, init["file"])
	if os.path.exists(init_file):
	subprocess.run(
	["docker", "cp", init_file, f"{container_id}:/tmp/init.sh"],
	env=env,
	)
	docker_exec(container_id, "bash /tmp/init.sh")

	start_cmd = task.get("start")
	if start_cmd:
	docker_exec(container_id, start_cmd, timeout=5)
	time.sleep(1)

	return container_id


	def stop_container(container_id):
	"""Stop a Docker container."""
	env = os.environ.copy()
	env["DOCKER_HOST"] = DOCKER_HOST
	subprocess.run(["docker", "kill", container_id], capture_output=True, env=env)


	def call_llm(backend, messages):
	"""Call the LLM backend and return the response.
	Handles both OpenAI-compatible and Ollama native API formats.
	"""
	config = BACKEND_CONFIG[backend]
	is_native = config.get("native", False)

	if is_native:
	# Ollama native API format
	payload = {
	"model": config["model"],
	"messages": messages,
	"stream": False,
	"think": False,
	"options": {"temperature": 0, "num_predict": 2048},
	}
	else:
	# OpenAI-compatible format
	payload = {
	"model": config["model"],
	"messages": messages,
	"temperature": 0,
	"max_tokens": 2048,
	}

	try:
	resp = requests.post(config["url"], json=payload, timeout=180)
	resp.raise_for_status()
	data = resp.json()

	if is_native:
	# Convert Ollama native response to OpenAI-compatible format
	msg = data.get("message", {})
	content = msg.get("content", "")
	prompt_tokens = data.get("prompt_eval_count", 0)
	eval_tokens = data.get("eval_count", 0)
	return {
	"choices": [{"message": {"role": "assistant", "content": content}}],
	"usage": {
	"prompt_tokens": prompt_tokens,
	"completion_tokens": eval_tokens,
	"total_tokens": prompt_tokens + eval_tokens,
	},
	}
	return data
	except requests.exceptions.ConnectionError:
	return {"error": f"Cannot connect to {backend} at {config['url']}"}
	except Exception as e:
	return {"error": str(e)}


	def parse_tool_calls(text):
	"""Parse tool calls from the model's text output.

	Handles multiple formats:
	1. JSON action format: {"action": "bash", "script": "ls"}
	2. Qwen tool_call tags: <tool_call>{"name": "bash_action", ...}</tool_call>
	3. Raw JSON with name/arguments
	"""
	# Strip thinking content (handles both closed and unclosed think tags)
	text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
	text = re.sub(r"<think>.*", "", text, flags=re.DOTALL) # unclosed
	text = re.sub(r"</think>", "", text) # orphaned closing
	text = re.sub(r"Thinking Process:.*?(?=\{\|<tool_call>\|$)", "", text, flags=re.DOTALL)

	tool_calls = []

	# Format 1: Find JSON objects (handles nested braces via balanced extraction)
	for start_idx in [m.start() for m in re.finditer(r'\{', text)]:
	depth = 0
	end_idx = start_idx
	in_string = False
	escape = False
	for i in range(start_idx, len(text)):
	c = text[i]
	if escape:
	escape = False
	continue
	if c == '\\':
	escape = True
	continue
	if c == '"' and not escape:
	in_string = not in_string
	continue
	if in_string:
	continue
	if c == '{':
	depth += 1
	elif c == '}':
	depth -= 1
	if depth == 0:
	end_idx = i + 1
	break
	if depth == 0 and end_idx > start_idx:
	candidate = text[start_idx:end_idx]
	try:
	obj = json.loads(candidate)
	action = obj.get("action", "")
	if action == "bash" and "script" in obj:
	tool_calls.append({"name": "bash_action", "arguments": {"script": obj["script"]}})
	elif action == "answer" and "value" in obj:
	tool_calls.append({"name": "answer_action", "arguments": {"answer": str(obj["value"])}})
	elif obj.get("name") in ("bash_action", "answer_action"):
	tool_calls.append(obj)
	if tool_calls:
	break # Take the first valid tool call
	except (json.JSONDecodeError, ValueError):
	pass

	if tool_calls:
	return tool_calls

	# Format 2: <tool_call>...</tool_call> blocks
	matches = re.findall(r"<tool_call>\s(.?)\s*</tool_call>", text, re.DOTALL)
	for m in matches:
	try:
	tc = json.loads(m.strip())
	tool_calls.append(tc)
	except json.JSONDecodeError:
	json_match = re.search(r"\{.*\}", m, re.DOTALL)
	if json_match:
	try:
	tc = json.loads(json_match.group())
	tool_calls.append(tc)
	except json.JSONDecodeError:
	pass

	return tool_calls


	def evaluate_task(task, answer, container_id):
	"""Evaluate the agent's answer against the task's oracle."""
	evaluation = task.get("evaluation", {})

	if "match" in evaluation:
	expected = evaluation["match"]
	if isinstance(expected, dict):
	if "answer" in expected:
	return answer.strip() == expected["answer"].strip()
	if "regex" in expected:
	return bool(re.search(expected["regex"], answer))
	else:
	return answer.strip() == str(expected).strip()

	if "check" in evaluation and "example" in evaluation:
	example = evaluation["example"]
	if isinstance(example, dict):
	example_code = example.get("code", "")
	else:
	example_code = str(example)
	if example_code and container_id:
	ground_truth = docker_exec(container_id, example_code, timeout=15)
	print(f" Ground truth: {ground_truth[:100]}")

	checks = evaluation["check"]
	for check in checks:
	if check is None:
	continue
	if isinstance(check, dict):
	lang = check.get("language", "python")
	check_file = check.get("file", "")
	check_path = os.path.join(SCRIPTS_DIR, check_file)

	if os.path.exists(check_path):
	try:
	if lang == "python":
	result = subprocess.run(
	["python3", check_path, answer.strip(), ground_truth.strip()],
	capture_output=True,
	timeout=10,
	)
	else:
	result = subprocess.run(
	["bash", check_path, answer.strip(), ground_truth.strip()],
	capture_output=True,
	timeout=10,
	)
	return result.returncode == 0
	except Exception as e:
	print(f" Check script error: {e}")
	return False

	print(f" WARNING: Could not evaluate")
	return False


	def run_task(task, task_idx, backend):
	"""Run a single AgentBench OS task."""
	description = task["description"]
	print(f"\n Task {task_idx}: {description[:80]}...")

	container_id = start_container(task)
	if not container_id:
	return {"task_idx": task_idx, "passed": False, "error": "container_start_failed", "rounds": 0, "latency": 0}

	messages = [
	{"role": "system", "content": SYSTEM_PROMPT},
	{"role": "user", "content": description},
	]

	answer = None
	total_tokens = 0
	start_time = time.time()

	try:
	for round_num in range(MAX_ROUNDS):
	print(f" Round {round_num + 1}/{MAX_ROUNDS}...", end=" ", flush=True)

	response = call_llm(backend, messages)
	if "error" in response:
	print(f"LLM error: {response['error']}")
	break

	usage = response.get("usage", {})
	total_tokens += usage.get("total_tokens", 0)

	choices = response.get("choices", [])
	if not choices:
	print("no choices")
	break

	content = choices[0].get("message", {}).get("content", "")
	if not content:
	print("empty response")
	break

	# Parse tool calls from the text
	tool_calls = parse_tool_calls(content)

	if not tool_calls:
	# No tool calls found — maybe the model answered directly
	clean = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
	clean = re.sub(r"<think>.*", "", clean, flags=re.DOTALL).strip()
	clean = re.sub(r"</think>", "", clean).strip()
	clean = re.sub(r"Thinking Process:.*", "", clean, flags=re.DOTALL).strip()
	if clean:
	# Try to extract just a number or short answer from verbose response
	lines = clean.strip().split("\n")
	last_line = lines[-1].strip()
	# If last line is a bare number, use it
	if re.match(r"^-?\d+\.?\d*$", last_line):
	answer = last_line
	else:
	answer = clean
	print(f"direct answer: {answer[:80]}")
	else:
	print("no tool calls and no useful content")
	messages.append({"role": "assistant", "content": content})
	break

	messages.append({"role": "assistant", "content": content})

	# Process tool calls
	tool_results = []
	for tc in tool_calls:
	name = tc.get("name", "")
	args = tc.get("arguments", {})

	if name == "answer_action":
	answer = str(args.get("answer", ""))
	print(f"ANSWER: {answer}")
	tool_results.append(f"Answer recorded: {answer}")
	break

	elif name == "bash_action":
	script = args.get("script", "")
	output = docker_exec(container_id, script)
	short_script = script[:50].replace('\n', ' ')
	short_output = output[:50].replace('\n', ' ')
	print(f"bash({short_script}) -> {short_output}")
	tool_results.append(output if output else "(no output)")
	else:
	print(f"unknown: {name}")
	tool_results.append(f"Unknown tool: {name}")

	# Feed tool results back
	if tool_results and answer is None:
	result_text = "\n".join(f"<tool_response>\n{r}\n</tool_response>" for r in tool_results)
	messages.append({"role": "user", "content": result_text})

	if answer is not None:
	break

	elapsed = time.time() - start_time

	if answer is not None:
	passed = evaluate_task(task, answer, container_id)
	print(f" Result: {'PASS' if passed else 'FAIL'} (answer={answer}, time={elapsed:.1f}s)")
	else:
	passed = False
	print(f" Result: FAIL (no answer, time={elapsed:.1f}s)")

	return {
	"task_idx": task_idx,
	"description": description[:100],
	"passed": passed,
	"answer": answer,
	"rounds": round_num + 1,
	"latency": round(elapsed, 2),
	"total_tokens": total_tokens,
	}

	finally:
	stop_container(container_id)


	def main():
	parser = argparse.ArgumentParser(description="AgentBench OS Evaluation Harness")
	parser.add_argument("--backend", choices=["ollama", "mlx-vlm", "llama-cpp"], required=True)
	parser.add_argument("--dataset", choices=["agentbench"], default="agentbench")
	parser.add_argument("--version", choices=["v1", "v2", "v3", "v4"], default="v1")
	parser.add_argument("--tasks", type=str, default=None, help="Comma-separated task indices")
	parser.add_argument("--model", type=str, default=None, help="Override model name (e.g. llama3.1:8b)")
	args = parser.parse_args()

	sample_file = os.path.join(BENCHMARKS_DIR, f"agentbench_os_{args.version}.json")
	with open(sample_file) as f:
	tasks = json.load(f)

	print(f"=== AgentBench OS {args.version} ===")
	print(f"Backend: {args.backend}")
	print(f"Tasks: {len(tasks)}")
	print(f"Endpoint: {BACKEND_CONFIG[args.backend]['url']}")

	# Check backend
	config = BACKEND_CONFIG[args.backend].copy()
	if args.model:
	config["model"] = args.model
	BACKEND_CONFIG[args.backend] = config
	try:
	if config.get("native"):
	resp = requests.get(config["url"].replace("/api/chat", "/api/tags"), timeout=5)
	else:
	base = config["url"].rsplit("/chat", 1)[0]
	resp = requests.get(base + "/models", timeout=5)
	print(f"Backend reachable: OK ({resp.status_code})")
	except Exception as e:
	print(f"WARNING: Backend check failed: {e}")

	if args.tasks:
	indices = [int(x) for x in args.tasks.split(",")]
	tasks_to_run = [(i, tasks[i]) for i in indices if i < len(tasks)]
	else:
	tasks_to_run = list(enumerate(tasks))

	results = []
	for task_idx, task in tasks_to_run:
	result = run_task(task, task_idx, args.backend)
	results.append(result)

	passed = sum(1 for r in results if r["passed"])
	total = len(results)
	avg_latency = sum(r["latency"] for r in results) / total if total else 0
	avg_tokens = sum(r["total_tokens"] for r in results) / total if total else 0

	print(f"\n{'='*50}")
	print(f"RESULTS: {args.backend} / agentbench_os_{args.version}")
	print(f"Passed: {passed}/{total}")
	print(f"Avg latency: {avg_latency:.1f}s")
	print(f"Avg tokens: {avg_tokens:.0f}")
	print(f"{'='*50}")

	model_tag = config["model"].replace("/", "_").replace(":", "_")
	results_dir = os.path.join(BENCHMARKS_DIR, "results", model_tag)
	os.makedirs(results_dir, exist_ok=True)
	results_file = os.path.join(results_dir, f"agentbench_os_{args.version}.json")
	print(f"Model: {config['model']}")
	with open(results_file, "w") as f:
	json.dump({
	"backend": args.backend,
	"dataset": "agentbench_os",
	"version": args.version,
	"model": config["model"],
	"passed": passed,
	"total": total,
	"avg_latency": round(avg_latency, 2),
	"avg_tokens": round(avg_tokens),
	"tasks": results,
	}, f, indent=2)

	print(f"Results saved to: {results_file}")
	return 0 if passed == total else 1


	if __name__ == "__main__":
	sys.exit(main())