Hugging Face's logo

Wellz1000
/
hermes-2

HERMES
LiteRT
ONNX
agent
telegram
ai-assistant
Model card Files
xet
 Community
hermes-2
File size: 4,250 Bytes
176d459
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/usr/bin/env python3
"""Compare enabled vs disabled runs and produce a readable report.



Reads scripts/out/_summary.json and the per-scenario JSONs, prints a side-by-

side comparison of what happened, and flags anomalies.

"""

from __future__ import annotations

import json
import sys
from pathlib import Path


HERE = Path(__file__).resolve().parent
OUT = HERE / "out"


def load_record(scenario_id: str, mode: str):
    path = OUT / f"{scenario_id}__{mode}.json"
    if not path.exists():
        return None
    return json.loads(path.read_text(encoding="utf-8"))


def fmt_tool_seq(calls):
    if not calls:
        return "(none)"
    return " β†’ ".join(c["name"] for c in calls)


def fmt_bridge_seq(calls):
    if not calls:
        return "(none)"
    parts = []
    for c in calls:
        if c["name"] == "tool_call":
            inner = (c.get("args") or {}).get("name", "?")
            parts.append(f"tool_call→{inner}")
        elif c["name"] == "tool_search":
            q = (c.get("args") or {}).get("query", "?")
            parts.append(f"search('{q[:30]}')")
        elif c["name"] == "tool_describe":
            n = (c.get("args") or {}).get("name", "?")
            parts.append(f"describe({n})")
    return " β†’ ".join(parts)


def main():
    if not OUT.exists():
        print("No output directory at", OUT)
        sys.exit(1)
    summary_path = OUT / "_summary.json"
    if not summary_path.exists():
        print("No _summary.json yet")
        sys.exit(1)

    summary = json.loads(summary_path.read_text(encoding="utf-8"))
    scenarios = sorted({row["scenario"] for row in summary})

    print(f"{'='*78}")
    print("  Live test results: tool_search ENABLED vs DISABLED")
    print(f"{'='*78}\n")

    fails = 0
    for sid in scenarios:
        en = load_record(sid, "enabled")
        di = load_record(sid, "disabled")
        if not en or not di:
            continue
        expected = set(en["expected_underlying_tools"])

        print(f"β”Œβ”€ {sid}  ({en['scenario_description']})")
        print(f"β”‚  Prompt: {en['prompt'][:120]}")
        print(f"β”‚  Expected underlying tools: {sorted(expected) or '(none)'}")
        print("β”‚")

        for label, rec in [("ENABLED ", en), ("DISABLED", di)]:
            called_under = [c["name"] for c in rec["underlying_tool_calls"]]
            called_set = set(called_under)
            missing = expected - called_set
            extra = called_set - expected - {"read_file", "search_files", "terminal", "todo", "memory"}

            mark = "βœ“" if (expected.issubset(called_set) and not rec["error"]) else "βœ—"
            if mark == "βœ—":
                fails += 1

            print(f"β”‚  {label} {mark}  bridges={len(rec['bridge_calls']):2}  underlying={len(rec['underlying_tool_calls']):2}  "
                  f"iters={rec['n_iterations']:2}  elapsed={rec['elapsed_seconds']:5.1f}s  err={bool(rec['error'])}")
            print(f"β”‚    underlying: {fmt_tool_seq(rec['underlying_tool_calls'])}")
            if rec["bridge_calls"]:
                print(f"β”‚    bridges:    {fmt_bridge_seq(rec['bridge_calls'])}")
            if missing:
                print(f"β”‚    ⚠ MISSING expected tools: {sorted(missing)}")
            if extra:
                print(f"β”‚    β“˜ extra tools called: {sorted(extra)}")
            if rec["error"]:
                print(f"β”‚    πŸ’₯ error: {rec['error'][:200]}")
        # Bridge-trip count vs direct (interesting comparator)
        en_bridges = len(en["bridge_calls"])
        di_underlying = len(di["underlying_tool_calls"])
        en_underlying = len(en["underlying_tool_calls"])
        overhead = en_bridges + en_underlying - di_underlying
        print(f"β”‚  Ξ” round-trip cost: enabled used {en_bridges + en_underlying} calls vs disabled {di_underlying}  β†’  +{overhead}")
        print(f"β”‚  Final (enabled):  {(en.get('final_response') or '')[:140]}")
        print(f"β”‚  Final (disabled): {(di.get('final_response') or '')[:140]}")
        print("└──")
        print()

    print(f"\nFails: {fails}/{2*len(scenarios)}")


if __name__ == "__main__":
    main()