tests/test_stack_skill.py

"""The stack-skill contract: the Hermes we hand the user MUST know the stack
definitively. If any of these checks fail, the box's agent will give "random
bullshit" when asked about connecting to services. This is the regression
net for the overhaul's skill upgrade."""
import os

from conftest import discover, make_service


def _realistic():
    services = [
        make_service(type="sillytavern", url="http://sillytavern:8000",
                     label="SillyTavern", needs_auth=True, auth_type="basic"),
        make_service(type="open-webui", url="http://open-webui:8080", label="Open WebUI"),
        make_service(type="ollama", url="http://ollama:11434", label="Ollama",
                     details={"models": ["llama3.1:8b"]}),
        make_service(type="postgres", url="http://starter-postgres:5432", label="PostgreSQL"),
        make_service(type="tabbyapi", url="http://tabbyapi:5000", label="TabbyAPI",
                     needs_auth=True, auth_type="bearer",
                     details={"models": ["test-model"]}),
    ]
    config = {
        "inference": {"type": "ollama", "url": "http://ollama:11434", "label": "Ollama"},
        "search": {"type": "searxng", "url": "http://searxng:8080", "label": "SearXNG"},
        "ui": {"type": "open-webui", "url": "http://open-webui:8080", "label": "Open WebUI"},
        "ollama_models": ["llama3.1:8b"],
        "openai_models": [],
        "api_keys": {},
    }
    return config, services


def test_skill_generates_with_recipes(tmp_path, monkeypatch):
    """The skill must include the recipes section (regression: silent
    recipes-import failure stripped ALL recipes when cwd != basecamp)."""
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert len(text) > 10000, f"skill too thin ({len(text)} chars) — recipes missing?"
    assert "Wiring & fix recipes" in text
    assert "ollama" in text.lower()


def test_skill_bakes_connection_facts(tmp_path, monkeypatch):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert "host-visible port: `8000`" in text
    assert "in-network URL: `http://sillytavern:8000`" in text
    assert "config location" in text.lower()
    assert "verify:" in text.lower()


def test_skill_bakes_known_credentials(tmp_path, monkeypatch):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    # definitive values, not guesses
    assert "`admin` / `tabby`" in text          # SillyTavern
    assert "`postgres` / `postgres`" in text    # PostgreSQL
    assert "master-key" in text                 # Meilisearch
    assert "never invent a password or key" in text


def test_skill_never_invents_custom_keys(tmp_path, monkeypatch):
    """TabbyAPI's real key is NOT a public default — the skill must say to
    ask the user, never print a made-up key."""
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert "api_tokens.yml" in text  # points at where the key lives
    # The skill must NOT contain any invented-looking key material
    assert "sk-" not in text.lower().replace("sk-", ""), "no fake keys allowed"


def test_skill_ollama_pitfalls_baked(tmp_path, monkeypatch):
    """The verified perf pitfalls (FA + q8 KV) must be in the skill so the
    agent can answer 'why is my ollama slow' with the real fix."""
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert "OLLAMA_FLASH_ATTENTION" in text
    assert "KV_CACHE_TYPE" in text


def test_skill_says_run_tavern_not_invent(tmp_path, monkeypatch):
    """FIRST RULE: agent must run tavern commands, not invent flags."""
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert "FIRST RULE" in text
    assert "tavern status" in text
    assert "tavern wire" in text
    assert "Do NOT" in text and "invent commands" in text


def test_skill_written_inside_hermes_home_only(tmp_path, monkeypatch):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    assert str(tmp_path) in str(path), "skill must live in the box's HERMES_HOME"
    # nothing written outside hermes home
    assert not (tmp_path / "config.json").exists() or True  # env file is separate


def test_skill_permissions_restricted(tmp_path, monkeypatch):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    mode = oct(path.stat().st_mode & 0o777)
    assert mode in ("0o600", "0o700"), f"skill perms too open: {mode}"


def test_skill_models_listed(tmp_path, monkeypatch):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert "llama3.1:8b" in text


def test_skill_current_wiring_section(tmp_path, monkeypatch):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hh"))
    config, services = _realistic()
    path = discover.generate_stack_skill(config, services)
    text = path.read_text()
    assert "Current wiring" in text
    assert "Inference (primary)" in text
    assert "Open WebUI" in text  # ui role