Eval v3: Hermes conv support + longest-block AST

eval_securecoder.py

@@ -1,4 +1,4 @@
-# /// script
+# /// script
 # requires-python = ">=3.10"
 # dependencies = [
 #     "datasets",
@@ -67,7 +67,7 @@ CODE_PROMPTS: list[str] = [
 
 TOOL_FN_PAT = re.compile(r"<function=([A-Za-z0-9_\.]+)>", re.S)
 TOOL_PARAM_PAT = re.compile(r"<parameter=([A-Za-z0-9_]+)>\s*(.*?)\s*</parameter>", re.S)
-
+
 
 
 def parse_args() -> argparse.Namespace:
@@ -81,7 +81,7 @@ def parse_args() -> argparse.Namespace:
     p.add_argument("--max-new-tokens", type=int, default=384)
     p.add_argument("--seed", type=int, default=3407)
     return p.parse_args()
-
+
 
 
 _TYPE_ALIASES = {
@@ -152,7 +152,7 @@ def _messages_from_any(row, kind):
         messages.append({"role": "assistant", "content": str(row.get("output") or row.get("response"))})
         return messages, tools
     return [], tools
-
+
 
 
 def _fetch_first_rows(repo: str, config: str | None, split: str, n: int) -> list[dict]:
@@ -170,10 +170,13 @@ def _fetch_first_rows(repo: str, config: str | None, split: str, n: int) -> list
 
 
 def _build_tool_prompts(rows: list[dict]) -> list[dict]:
+    """Build (prompt, tools) tuples from a mix of Hermes-style ``conversations``
+    (with ``from``/``value`` keys) and OpenAI-style ``messages`` rows.
+    The Hermes rows are by far the more common in NousResearch/hermes-function-calling-v1."""
+    ROLE_MAP = {"human": "user", "system": "system", "gpt": "assistant",
+                "user": "user", "assistant": "assistant"}
     prompts = []
     for row in rows:
-        if not isinstance(row.get("messages"), list):
-            continue
         tools_raw = row.get("tools")
         if not tools_raw:
             continue
@@ -189,19 +192,27 @@ def _build_tool_prompts(rows: list[dict]) -> list[dict]:
                 tools.append(n)
         if not tools:
             continue
-        messages, _ = _messages_from_any(row, "auto")
-        if not messages:
-            continue
-        user = next((m["content"] for m in messages if m["role"] == "user"), None)
-        if not user:
+
+        user_content = None
+        convo = row.get("messages") or row.get("conversations") or []
+        if isinstance(convo, list):
+            for turn in convo:
+                if not isinstance(turn, dict):
+                    continue
+                role_raw = str(turn.get("from") or turn.get("role") or "").lower()
+                role = ROLE_MAP.get(role_raw)
+                if role != "user":
+                    continue
+                user_content = turn.get("value", turn.get("content", ""))
+                break
+        if not user_content:
             continue
-        prompts.append({"prompt": str(user)[:1200], "tools": tools,
-                        "expected_call": any(m.get("tool_calls") for m in messages)})
+
+        prompts.append({"prompt": str(user_content)[:1200], "tools": tools,
+                        "expected_call": True})
         if len(prompts) >= 200:
             break
     return prompts
-
-
 def _render_prompt(tokenizer, prompt: str, tools: list[dict]) -> str:
     return tokenizer.apply_chat_template(
         [{"role": "user", "content": prompt}],
@@ -236,7 +247,7 @@ def _score_call(call: dict, tools: list[dict]) -> dict:
     return {"parse": True, "name_ok": True,
             "schema_ok": expected.issubset(given) if expected else True,
             "expected_keys": sorted(expected), "given_keys": sorted(given)}
-
+
 
 
 def eval_tool_calls(tokenizer, model, args) -> dict:
@@ -294,26 +305,34 @@ def eval_code_sanity(tokenizer, model, args) -> dict:
             out.append({"prompt": prompt[:60], "error": repr(exc)[:120]})
             continue
         block = None
-        m = re.search(r"```(?:python)?\s*\n(.*?)```", reply, re.S)
-        if m:
-            block = m.group(1)
+        # Truncation inside a code block is common (max_new_tokens cuts
+        # mid-line), so we keep the LONGEST ```python ... ``` block in the
+        # reply rather than the first one. Most failures in the first eval
+        # run were valid code cut mid-comment, not bad code.
+        blocks = re.findall(r"```(?:python)?\s*\n(.*?)```", reply, re.S)
+        if blocks:
+            block = max(blocks, key=len)
         else:
             start = reply.find("def ")
             if start >= 0:
                 block = reply[start:]
+        if block:
+            # Strip anything past a closing fence on the same line
+            block = re.split(r"\n```", block, maxsplit=1)[0]
+
         parsed = compiles = None
         if block:
             try:
-                ast.parse(block); parsed = True
+                ast.parse(block)
+                parsed = True
             except SyntaxError:
-                parsed = False; block = None
-        if block:
+                parsed = False
+        if block and parsed:
             try:
-                compile(block, "<eval>", "exec"); compiles = True
+                compile(block, "<eval>", "exec")
+                compiles = True
             except Exception:
                 compiles = False
-        out.append({"prompt": prompt[:60], "ast_ok": parsed, "compile_ok": compiles,
-                    "reply_first_160": reply[:160]})
         if (i + 1) % 5 == 0:
             log.info("  code sanity: %d/%d", i + 1, len(prompts))
     n = len(out)
@@ -322,7 +341,7 @@ def eval_code_sanity(tokenizer, model, args) -> dict:
     return {"section": "code_sanity", "n_prompts": n,
             "ast_rate": ast_ok / max(n, 1), "compile_rate": compile_ok / max(n, 1),
             "details": out}
-
+
 
 
 def eval_security_mcq(tokenizer, model, n_questions: int = 25) -> dict: