Eval v4: Hermes JSON-string tools, merged-repo loader, eos_token_id

eval_securecoder.py

@@ -5,8 +5,7 @@
 #     "huggingface_hub",
 #     "torch",
 #     "transformers",
-#     "peft",
-#     "unsloth",
+#     "bitsandbytes",
 # ]
 # ///
 """Cheap evaluations for SecureCoder.
@@ -94,18 +93,39 @@ _TYPE_ALIASES = {
 
 
 def _normalise_tool_schema(tool):
+    """Accept tool specs as a dict, a list of dicts, or a JSON string. Hermes
+    ships ``tools`` as a JSON string; xLAM and most others use a list/dict."""
+    if tool is None:
+        return None
+    if isinstance(tool, str):
+        try:
+            tool = json.loads(tool)
+        except (json.JSONDecodeError, TypeError):
+            return None
+    if isinstance(tool, list):
+        for entry in tool:
+            norm = _normalise_tool_schema(entry)
+            if norm:
+                return norm
+        return None
     if not isinstance(tool, dict):
         return None
     fn = tool.get("function") if isinstance(tool.get("function"), dict) else tool
     if not isinstance(fn, dict) or not fn.get("name"):
         return None
     params = fn.get("parameters") or {}
+    if isinstance(params, str):
+        try:
+            params = json.loads(params)
+        except (json.JSONDecodeError, TypeError):
+            params = {}
     if isinstance(params, dict) and params and "properties" not in params:
         properties: dict[str, Any] = {}
         required: list[str] = []
         for name, spec in params.items():
             if isinstance(spec, dict):
-                cleaned = {k: v for k, v in spec.items() if k in ("type", "description", "enum", "default", "title", "items")}
+                cleaned = {k: v for k, v in spec.items()
+                           if k in ("type", "description", "enum", "default", "title", "items")}
                 cleaned["type"] = _TYPE_ALIASES.get(str(cleaned.get("type", "")).lower(), "string")
                 properties[name] = cleaned
                 if "default" not in spec:
@@ -120,95 +140,55 @@ def _normalise_tool_schema(tool):
 
 
 def _messages_from_any(row, kind):
+    """Extract a single user turn from a Hermes ``conversations`` row (from/value)
+    or an OpenAI-style ``messages`` row (role/content). Tools can be a list, dict,
+    or JSON string - we keep the first valid schema."""
+    tools_raw = row.get("tools")
+    if isinstance(tools_raw, str):
+        try:
+            tools_raw = json.loads(tools_raw)
+        except (json.JSONDecodeError, TypeError):
+            tools_raw = None
     tools: list[dict] = []
-    raw = row.get("tools")
-    candidates = [raw] if isinstance(raw, dict) else (raw if isinstance(raw, list) else [])
+    candidates = tools_raw if isinstance(tools_raw, list) else ([tools_raw] if isinstance(tools_raw, dict) else [])
     for c in candidates:
         n = _normalise_tool_schema(c)
         if n:
             tools.append(n)
+
     messages: list[dict] = []
-    if isinstance(row.get("messages"), list):
-        for turn in row["messages"]:
+    convo = row.get("messages") or row.get("conversations") or []
+    if isinstance(convo, str):
+        try:
+            convo = json.loads(convo)
+        except (json.JSONDecodeError, TypeError):
+            convo = []
+    ROLE_MAP = {"human": "user", "system": "system", "gpt": "assistant",
+                "user": "user", "assistant": "assistant"}
+    if isinstance(convo, list):
+        for turn in convo:
             if not isinstance(turn, dict):
                 continue
-            role = turn.get("role")
-            content = turn.get("content", "")
-            if role in {"user", "assistant", "system"} and content:
-                messages.append({"role": role, "content": str(content)})
-        return messages, tools
-    if row.get("query") and row.get("answers"):
-        messages.append({"role": "user", "content": str(row["query"])})
-        messages.append({"role": "assistant", "content": str(row["answers"])})
-        return messages, tools
-    if row.get("user") and row.get("assistant"):
-        if row.get("system"):
-            messages.append({"role": "system", "content": str(row["system"])})
-        messages.append({"role": "user", "content": str(row["user"])})
-        messages.append({"role": "assistant", "content": str(row["assistant"])})
-        return messages, tools
-    if row.get("instruction") and (row.get("output") or row.get("response")):
-        messages.append({"role": "user", "content": str(row["instruction"])})
-        messages.append({"role": "assistant", "content": str(row.get("output") or row.get("response"))})
-        return messages, tools
-    return [], tools
-
-
-
-def _fetch_first_rows(repo: str, config: str | None, split: str, n: int) -> list[dict]:
-    from datasets import load_dataset
-    kwargs: dict[str, Any] = {"split": split, "streaming": True}
-    if config:
-        kwargs["name"] = config
-    ds = load_dataset(repo, token=os.environ.get("HF_TOKEN"), **kwargs)
-    out = []
-    for row in ds:
-        out.append(dict(row))
-        if len(out) >= n:
-            break
-    return out
-
-
+            role_raw = str(turn.get("from") or turn.get("role") or "").lower()
+            role = ROLE_MAP.get(role_raw)
+            if role == "user":
+                content = turn.get("value", turn.get("content", ""))
+                messages.append({"role": "user", "content": str(content)})
+                break
+    return messages, tools
 def _build_tool_prompts(rows: list[dict]) -> list[dict]:
-    """Build (prompt, tools) tuples from a mix of Hermes-style ``conversations``
-    (with ``from``/``value`` keys) and OpenAI-style ``messages`` rows.
-    The Hermes rows are by far the more common in NousResearch/hermes-function-calling-v1."""
-    ROLE_MAP = {"human": "user", "system": "system", "gpt": "assistant",
-                "user": "user", "assistant": "assistant"}
+    """Build (prompt, tools) tuples via the shared _messages_from_any helper.
+    Hermes rows use ``conversations`` (from/value); OpenAI-style use ``messages``
+    (role/content). Both are accepted transparently."""
     prompts = []
     for row in rows:
-        tools_raw = row.get("tools")
-        if not tools_raw:
-            continue
-        tools: list[dict] = []
-        if isinstance(tools_raw, list):
-            for t in tools_raw:
-                n = _normalise_tool_schema(t)
-                if n:
-                    tools.append(n)
-        elif isinstance(tools_raw, dict):
-            n = _normalise_tool_schema(tools_raw)
-            if n:
-                tools.append(n)
-        if not tools:
+        messages, tools = _messages_from_any(row, "auto")
+        if not tools or not messages:
             continue
-
-        user_content = None
-        convo = row.get("messages") or row.get("conversations") or []
-        if isinstance(convo, list):
-            for turn in convo:
-                if not isinstance(turn, dict):
-                    continue
-                role_raw = str(turn.get("from") or turn.get("role") or "").lower()
-                role = ROLE_MAP.get(role_raw)
-                if role != "user":
-                    continue
-                user_content = turn.get("value", turn.get("content", ""))
-                break
-        if not user_content:
+        user = next((m["content"] for m in messages if m["role"] == "user"), None)
+        if not user:
             continue
-
-        prompts.append({"prompt": str(user_content)[:1200], "tools": tools,
+        prompts.append({"prompt": str(user)[:1200], "tools": tools,
                         "expected_call": True})
         if len(prompts) >= 200:
             break
@@ -266,7 +246,7 @@ def eval_tool_calls(tokenizer, model, args) -> dict:
             text = _render_prompt(tokenizer, p["prompt"], p["tools"])
             ids = tokenizer(text, return_tensors="pt", add_special_tokens=False).input_ids.to(model.device)
             with torch.no_grad():
-                generated = model.generate(ids, max_new_tokens=args.max_new_tokens, do_sample=False)
+                generated = model.generate(ids, max_new_tokens=args.max_new_tokens, do_sample=False, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.eos_token_id)
             reply = tokenizer.decode(generated[0, ids.shape[-1]:], skip_special_tokens=True)
         except Exception as exc:
             out.append({"prompt": p["prompt"][:60], "error": repr(exc)[:120]})
@@ -299,7 +279,7 @@ def eval_code_sanity(tokenizer, model, args) -> dict:
         ids = tokenizer(text, return_tensors="pt", add_special_tokens=False).input_ids.to(model.device)
         try:
             with torch.no_grad():
-                generated = model.generate(ids, max_new_tokens=384, do_sample=False)
+                generated = model.generate(ids, max_new_tokens=384, do_sample=False, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.eos_token_id)
             reply = tokenizer.decode(generated[0, ids.shape[-1]:], skip_special_tokens=True)
         except Exception as exc:
             out.append({"prompt": prompt[:60], "error": repr(exc)[:120]})
@@ -372,7 +352,7 @@ def eval_security_mcq(tokenizer, model, n_questions: int = 25) -> dict:
         ids = tokenizer(text, return_tensors="pt", add_special_tokens=False).input_ids.to(model.device)
         try:
             with torch.no_grad():
-                generated = model.generate(ids, max_new_tokens=8, do_sample=False)
+                generated = model.generate(ids, max_new_tokens=8, do_sample=False, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.eos_token_id)
             reply = tokenizer.decode(generated[0, ids.shape[-1]:], skip_special_tokens=True).strip()
         except Exception:
             continue
@@ -448,3 +428,7 @@ def main() -> int:
 
 if __name__ == "__main__":
     raise SystemExit(main())
+
+
+
+