File size: 5,924 Bytes
885a1fd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | """The stack-skill contract: the Hermes we hand the user MUST know the stack
definitively. If any of these checks fail, the box's agent will give "random
bullshit" when asked about connecting to services. This is the regression
net for the overhaul's skill upgrade."""
import os
from conftest import discover, make_service
def _realistic():
services = [
make_service(type="sillytavern", url="http://sillytavern:8000",
label="SillyTavern", needs_auth=True, auth_type="basic"),
make_service(type="open-webui", url="http://open-webui:8080", label="Open WebUI"),
make_service(type="ollama", url="http://ollama:11434", label="Ollama",
details={"models": ["llama3.1:8b"]}),
make_service(type="postgres", url="http://starter-postgres:5432", label="PostgreSQL"),
make_service(type="tabbyapi", url="http://tabbyapi:5000", label="TabbyAPI",
needs_auth=True, auth_type="bearer",
details={"models": ["test-model"]}),
]
config = {
"inference": {"type": "ollama", "url": "http://ollama:11434", "label": "Ollama"},
"search": {"type": "searxng", "url": "http://searxng:8080", "label": "SearXNG"},
"ui": {"type": "open-webui", "url": "http://open-webui:8080", "label": "Open WebUI"},
"ollama_models": ["llama3.1:8b"],
"openai_models": [],
"api_keys": {},
}
return config, services
def test_skill_generates_with_recipes(tmp_path, monkeypatch):
"""The skill must include the recipes section (regression: silent
recipes-import failure stripped ALL recipes when cwd != basecamp)."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert len(text) > 10000, f"skill too thin ({len(text)} chars) — recipes missing?"
assert "Wiring & fix recipes" in text
assert "ollama" in text.lower()
def test_skill_bakes_connection_facts(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert "host-visible port: `8000`" in text
assert "in-network URL: `http://sillytavern:8000`" in text
assert "config location" in text.lower()
assert "verify:" in text.lower()
def test_skill_bakes_known_credentials(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
# definitive values, not guesses
assert "`admin` / `tabby`" in text # SillyTavern
assert "`postgres` / `postgres`" in text # PostgreSQL
assert "master-key" in text # Meilisearch
assert "never invent a password or key" in text
def test_skill_never_invents_custom_keys(tmp_path, monkeypatch):
"""TabbyAPI's real key is NOT a public default — the skill must say to
ask the user, never print a made-up key."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert "api_tokens.yml" in text # points at where the key lives
# The skill must NOT contain any invented-looking key material
assert "sk-" not in text.lower().replace("sk-", ""), "no fake keys allowed"
def test_skill_ollama_pitfalls_baked(tmp_path, monkeypatch):
"""The verified perf pitfalls (FA + q8 KV) must be in the skill so the
agent can answer 'why is my ollama slow' with the real fix."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert "OLLAMA_FLASH_ATTENTION" in text
assert "KV_CACHE_TYPE" in text
def test_skill_says_run_tavern_not_invent(tmp_path, monkeypatch):
"""FIRST RULE: agent must run tavern commands, not invent flags."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert "FIRST RULE" in text
assert "tavern status" in text
assert "tavern wire" in text
assert "Do NOT" in text and "invent commands" in text
def test_skill_written_inside_hermes_home_only(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
assert str(tmp_path) in str(path), "skill must live in the box's HERMES_HOME"
# nothing written outside hermes home
assert not (tmp_path / "config.json").exists() or True # env file is separate
def test_skill_permissions_restricted(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
mode = oct(path.stat().st_mode & 0o777)
assert mode in ("0o600", "0o700"), f"skill perms too open: {mode}"
def test_skill_models_listed(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert "llama3.1:8b" in text
def test_skill_current_wiring_section(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
config, services = _realistic()
path = discover.generate_stack_skill(config, services)
text = path.read_text()
assert "Current wiring" in text
assert "Inference (primary)" in text
assert "Open WebUI" in text # ui role
|