Buckets:

download
raw
51.8 kB
""" Pytest configuration and shared test utilities. This module provides shared fixtures and utilities for all Osprey tests. """ import logging import os import time from dataclasses import dataclass, field from pathlib import Path import pytest from rich.logging import RichHandler from osprey.utils.logger import QUIET_THIRD_PARTY_LOGGERS from tests import ci_diagnostics #: Repo root — the fallback when a test leaves the process in a deleted cwd. _REPO_ROOT = Path(__file__).resolve().parent.parent #: Root-logging state as it was before collection — see ``restore_root_logging``. #: Captured at conftest import, which is the only moment guaranteed to precede #: every test *and* every higher-scoped fixture. _PRISTINE_LOGGING = { "root_level": logging.getLogger().level, "root_handlers": list(logging.getLogger().handlers), "third_party": {name: logging.getLogger(name).level for name in QUIET_THIRD_PARTY_LOGGERS}, } # =================================================================== # Collection-time environment pollution guard # =================================================================== # # Importing a test module can import third-party code that loads a .env at # import time — litellm calls dotenv.load_dotenv() on import, and python-dotenv # walks UP from its package directory, so on a developer machine it can find an # ancestor checkout's .env and inject real credentials, PROJECT_ROOT, and TZ # into os.environ before the first test runs. CI has no ancestor .env, so # whatever those variables change locally is invisible there. # # Only that injection is undone. A blanket restore would also strip the # coordination variables some test modules legitimately export at import time # (tests/va/* set EPICS_CA_SERVER_PORT et al. so the soft-IOC subprocesses they # spawn share their ports), so a key is removed only when it BOTH appeared # during collection AND carries exactly the value an ancestor .env defines. # import-time required because `osprey.cli.styles` builds its module-level # Rich Console at import time and Rich reads FORCE_COLOR in `Console.__init__`, # so a fixture runs too late to un-force a console that already exists: the # color-forcing variables must be gone BEFORE collection imports any osprey # module. A developer terminal that # exports FORCE_COLOR otherwise turns CliRunner captures into ANSI-laced text # and fails seven CLI tests on pristine main — CI is green only because its # runners never export it. `NO_COLOR` is left alone: unset is the same default # CI runs under. os.environ.pop("FORCE_COLOR", None) os.environ.pop("CLICOLOR_FORCE", None) # Rich deliberately clamps ``TERM=dumb`` consoles to 80 columns and suppresses # cursor control even when a test explicitly requests ``force_terminal=True``. # CI presents a capable TERM, so normalize only the dumb/unknown host values; # tests for a restricted terminal can still set one explicitly after collection. if os.environ.get("TERM", "").lower() in {"dumb", "unknown"}: os.environ["TERM"] = "xterm-256color" _PRE_COLLECTION_ENV = dict(os.environ) def _ancestor_dotenv_values() -> dict[str, str]: """Every assignment an ancestor .env could have injected, keyed by var.""" from dotenv import dotenv_values injected: dict[str, str] = {} for directory in (_REPO_ROOT, *_REPO_ROOT.parents): candidate = directory / ".env" if candidate.is_file(): injected.update({k: v for k, v in dotenv_values(candidate).items() if v is not None}) return injected def pytest_collection_finish(session): """Undo ancestor-.env injection performed by imports during collection.""" import time added = set(os.environ) - set(_PRE_COLLECTION_ENV) if added: injected = _ancestor_dotenv_values() for key in added: if key in injected and os.environ[key] == injected[key]: del os.environ[key] # Re-sync the C library's cached zone with the (possibly restored) TZ. # Collection can leave the two disagreeing (TZ injected, cache not), and # then the first test to call tzset() flips tzname mid-suite and reads as # a leaker to _no_host_timezone_leak below. time.tzset() # =================================================================== # Environment guard # =================================================================== # # Ordering note: autouse fixtures of equal scope are set up in *alphabetical* # order, not declaration order — `pytest --setup-plan <test>` prints the real # sequence. So `_no_host_timezone_leak` is the outermost function-scoped fixture # here (leading underscore), not this one, and anything that must sit outside # that guard's snapshot window has to be session-scoped rather than merely # declared earlier. @pytest.fixture(autouse=True, scope="function") def restore_environ(): """Snapshot and restore the whole of ``os.environ`` around every test. Leak guarded: plenty of code writes ``os.environ`` directly rather than through ``monkeypatch`` — ``.env`` loading in the health CLI and the config loader uses override semantics, and the web terminal lifespan assigns ``OSPREY_CONFIG`` by hand. Without a full snapshot those writes survive into sibling tests, which under xdist makes results depend on how the files happened to be distributed. ``OSPREY_CONFIG`` and ``CONFIG_FILE`` are additionally cleared on the way in, so a developer who exports either in their shell still gets a pristine run. ``TZ`` is handled once per session instead -- see ``_clear_dotenv_timezone``. """ saved = dict(os.environ) os.environ.pop("OSPREY_CONFIG", None) os.environ.pop("CONFIG_FILE", None) try: yield finally: os.environ.clear() os.environ.update(saved) @pytest.fixture(autouse=True, scope="function") def session_posture_leak_guard(monkeypatch): """Run every test outside a web-terminal session's write posture. Three variables carry that posture: ``OSPREY_EXECUTION_MODE`` (a read-only run), and the two anchors ``OSPREY_POSTURE_SESSION`` and ``OSPREY_AGENT_DATA_ROOT``, which the web server always stamps as a pair. A developer running the suite from inside a narrowed session would otherwise hand every record-touching test a session id and an agent-data root no test asked for — and, worse, a test that redirects the record by patching ``resolve_shared_data_root`` would be silently inert, because ``posture_store.agent_data_root()`` reads the variable FIRST and only falls back to the resolver. All three are CLEARED, not pointed elsewhere. Stamping a throwaway root here would look tidier — clearing the variable does not stop a write, it aims it at ``<repo>/var/agent_data`` via ``resolve_shared_data_root()`` — but the stamp is preferred over the resolver by both readers, so a suite-wide stamp silently overrides every test that redirects the root by patching ``resolve_shared_data_root``, which is most of them: measured, it turns 90 tests across ``tests/mcp_server`` red and the whole hooks tree with it. The leak is therefore closed where it is caused — a test whose code writes under that root stamps its own — and kept closed by :func:`no_agent_data_in_the_repo` below, which fails the session if the run created the directory. Named without a leading underscore on purpose. Autouse fixtures of equal scope are set up in alphabetical order, so this sorts AFTER ``restore_environ`` and is therefore torn down before it — a name like ``_no_session_posture`` would delete the variables before that snapshot was taken, and the restore would then drop them for the rest of the session. """ for anchor in ( "OSPREY_EXECUTION_MODE", "OSPREY_POSTURE_SESSION", "OSPREY_AGENT_DATA_ROOT", ): monkeypatch.delenv(anchor, raising=False) yield _REAL_DEPLOYMENT_LANES = ("tests/e2e/", "tests/va/e2e/") @pytest.fixture(autouse=True, scope="session") def no_agent_data_in_the_repo(request): """Fail the session if the suite created ``<repo>/var/agent_data``. The regression this exists for is silent by construction: ``var/`` is gitignored, the marker files are unlinked on the way out, and what is left behind is an empty directory that ``git status`` never mentions. It was found twice by hand; this is what finds it the third time. This is the guard that actually holds the line, rather than a suite-wide stamp of ``OSPREY_AGENT_DATA_ROOT`` — see :func:`session_posture_leak_guard`. A stamp would silence the symptom for every test at once, at the cost of overriding the resolver patch most store-touching tests use to redirect that root. Only a directory the RUN created is a failure. One that was already there belongs to a real local deployment and is none of the suite's business — checking for creation rather than existence is what keeps this from firing on a developer who has actually run OSPREY in this checkout. The real-deployment lanes (``tests/e2e/``, ``tests/va/e2e/``) are exempt: they run agents and servers with this checkout as the project root, so the executor's run folders land under ``var/agent_data`` by design, not by a fixture's mistake. The guard is armed only in a session that collects none of them — the unit lane it was written for. """ marker = Path(__file__).resolve().parent.parent / "var" / "agent_data" real_deployment_lane = any( item.nodeid.startswith(_REAL_DEPLOYMENT_LANES) for item in request.session.items ) existed = marker.exists() yield if real_deployment_lane: return if not existed and marker.exists(): raise AssertionError( f"the test run created {marker} — something resolved the agent-data root " "to the repository. A test that writes the posture store or a control-target " "state file must stamp OSPREY_AGENT_DATA_ROOT at a tmp path (see " "session_posture_leak_guard) rather than leave it to resolve_shared_data_root()." ) # =================================================================== # Auto-reset Registry and Config Between Tests # =================================================================== @pytest.fixture(autouse=True, scope="function") def reset_state_between_tests(): """Auto-reset registry and config before each test to ensure isolation. This prevents state leakage between tests by: - Resetting the registry - Clearing config caches (utils.config and utils.workspace) Config-related environment variables are handled by ``restore_environ``. """ # Reset before test from osprey.registry import reset_registry from osprey.utils.workspace import reset_config_cache reset_registry() reset_config_cache() yield # Reset after test reset_registry() reset_config_cache() # =================================================================== # Web-auth test seam # =================================================================== # # ``WebAuthMiddleware`` is installed outermost on every interface app by # ``configure_interface_app``, so every request a ``starlette.testclient`` # (a.k.a. ``fastapi.testclient``) ``TestClient`` — or an ``httpx.AsyncClient`` # wrapping an ``ASGITransport`` — makes is 401'd unless it carries a live # operator credential. Hundreds of existing tests drive interface routes through # these clients and predate the gate; they assert the route's own behaviour, not # the absence of authentication. Rather than touch each of them, this seam makes # every such client present the operator credential by default. # # Mechanism — request-time operator-secret injection, *not* a construction-time # cookie. Two facts about the existing suite force this: # # 1. Some interface tests pin their own ``app.state.web_credentials`` (a fresh # :class:`WebCredentials` with a known operator secret) *after* the client # is constructed — ``tests/interfaces/web_terminal/test_proxy.py`` does. A # credential minted at client-construction time lands in the holder that # swap discards, so it never authenticates. Reading the holder on each # request, instead, always sees the live one. # 2. The proxy-boundary tests attach *decoy* inbound headers (a wrong # ``X-Osprey-Terminal-Secret``, a stale cookie, a bearer) to prove the # proxy strips them. The gate checks the operator-secret header *first* and # refuses a wrong one outright, so a session cookie added alongside a decoy # secret header would never be reached. Overriding that header with the # process's real secret is the only thing that authenticates such a request. # # So the seam force-sets ``X-Osprey-Terminal-Secret`` (``OPERATOR_SECRET_HEADER``) # to the operator secret held by the client's target app, per request, reading # ``app.state.web_credentials`` at send time. This is the header path the task # explicitly allows as the alternative to the cookie: the cookie path is the one # real browsers use, but it cannot serve these two shapes, and the Origin/CSRF # logic it would exercise is already covered directly by the raw-ASGI tests in # ``tests/interfaces/test_auth_middleware.py``. The operator-secret header is not # subject to the Origin check, so mutating and websocket requests pass too. # # The seam acts **only** on a client aimed at a gated interface app — one whose # ``app.state.web_credentials`` is a real :class:`WebCredentials` (the object the # gate reads, and the tell that ``configure_interface_app`` ran). A client aimed # at any other ASGI app, or at a real network host, is left untouched. Both # client shapes are covered: ``TestClient`` over HTTP *and* its # ``websocket_connect`` handshake, and ``httpx.AsyncClient`` over ``ASGITransport``. # # Install is session-scoped so it wraps client fixtures of every scope — a # module-scoped client is built before any function-scoped fixture would run. # Opt-out is a per-test flag (``@pytest.mark.no_auth_seam``) the injection hooks # read at send time, for the dedicated tests that assert an *unauthenticated* # refusal through such a client. The raw-ASGI auth tests never build one, so they # need no marker. #: Flipped off by :func:`_auth_seam_optout` for a ``no_auth_seam`` test. Read by #: the injection hooks at send time, so it governs even a client built by a #: higher-scoped fixture and shared across tests. _AUTH_SEAM_ON = True @pytest.fixture(autouse=True, scope="function") def reset_web_credentials_between_tests(monkeypatch: pytest.MonkeyPatch): """Forget the process web credentials before and after every test. The env-driven population path (``web_auth._populate``) is decided once per process and is idempotent, so without this the first test in a worker would pin the operator secret for every test after it, and a test that mints a secret would leak it into its neighbours. Resetting both sides keeps the population testable in isolation. It does not clear ``app.state`` on an app built before the reset — such an app keeps the holder it cached. The root TestClient seam reads its secret from ``app.state``, so that app stays reachable through it; the browser and live-server seams in ``tests/interfaces/conftest.py`` read the PROCESS holder instead, so a module-level app must be re-pointed at it with ``use_process_web_credentials(app)`` before those seams can authenticate. Also clears ``SESSION_LIFETIME_ENV`` and ``SESSION_STORE_DIR_ENV`` before every test, for the same leak-prevention reason: ``tests/cli/test_discovery_ rewire.py`` runs the real ``osprey web`` command in-process, and the launcher publishes both ``OSPREY_TERMINAL_SESSION_LIFETIME`` and ``OSPREY_TERMINAL_SESSION_STORE_DIR`` into ``os.environ`` for the child process to read. Left in place, every later test sharing that worker would populate a holder pointed at a real, on-disk store directory instead of the unconfigured one most tests assume. ``monkeypatch`` restores whichever value (if any) was present before the test once it tears down, rather than each of the two sides here reset unconditionally the way the credentials calls above do. """ from osprey.interfaces.web_auth import ( SESSION_LIFETIME_ENV, SESSION_STORE_DIR_ENV, reset_web_credentials, ) from osprey.mcp_server.http import reset_panel_token_latch monkeypatch.delenv(SESSION_LIFETIME_ENV, raising=False) monkeypatch.delenv(SESSION_STORE_DIR_ENV, raising=False) # The MCP client latches the last panel token it saw (so an in-process # companion launch scrubbing the carrier does not strip its bearer); that # latch is process state for the same reason and is reset on both sides too. reset_web_credentials() reset_panel_token_latch() yield reset_web_credentials() reset_panel_token_latch() def _gated_interface_app(client): """Return the client's target app if it is a gated interface app, else None. A ``TestClient`` records the app on ``self.app``; an httpx client wrapping an ``ASGITransport`` carries it on the transport. The tell that the app installed :class:`WebAuthMiddleware` is a real :class:`WebCredentials` on ``app.state`` — seeded by ``configure_interface_app`` and read by the gate. """ from osprey.interfaces.web_auth import WebCredentials app = getattr(client, "app", None) if app is None: app = getattr(getattr(client, "_transport", None), "app", None) credentials = getattr(getattr(app, "state", None), "web_credentials", None) return app if isinstance(credentials, WebCredentials) else None def _current_operator_secret(app): """The operator secret ``app``'s gate will accept right now, or None.""" from osprey.interfaces.web_auth import WebCredentials credentials = getattr(getattr(app, "state", None), "web_credentials", None) if isinstance(credentials, WebCredentials): return credentials.operator_secret return None def _install_client_auth(client) -> None: """Arm one httpx client to present the operator secret to its gated app. A no-op for a client not aimed at a gated interface app. Otherwise it appends a request event hook (async for an ``AsyncClient``, sync otherwise) that force-sets the operator-secret header per request, and — for a client that speaks websockets — wraps ``websocket_connect`` to carry the same header on the handshake, which the request hooks never see. """ import httpx app = _gated_interface_app(client) if app is None: return from osprey.interfaces.common_middleware import OPERATOR_SECRET_HEADER def _apply(request) -> None: if not _AUTH_SEAM_ON: return secret = _current_operator_secret(app) if secret: request.headers[OPERATOR_SECRET_HEADER] = secret async def _apply_async(request) -> None: _apply(request) hook = _apply_async if isinstance(client, httpx.AsyncClient) else _apply client.event_hooks.setdefault("request", []).append(hook) ws_connect = getattr(client, "websocket_connect", None) if callable(ws_connect): def _seamed_ws(url, *args, **kwargs): if _AUTH_SEAM_ON: secret = _current_operator_secret(app) if secret: merged = dict(kwargs.get("headers") or {}) merged.setdefault(OPERATOR_SECRET_HEADER, secret) kwargs["headers"] = merged return ws_connect(url, *args, **kwargs) client.websocket_connect = _seamed_ws @pytest.fixture(autouse=True, scope="session") def _install_auth_seam(): """Patch the client constructors once so every client fixture is covered. Session-scoped on purpose: a module- or class-scoped client fixture is built before any function-scoped fixture runs, so a function-scoped patch would miss it. The per-test opt-out lives in :data:`_AUTH_SEAM_ON`, which the injection hooks read at send time. """ import httpx from starlette.testclient import TestClient test_client_init = TestClient.__init__ async_client_init = httpx.AsyncClient.__init__ def _seamed_test_client_init(self, *args, **kwargs): test_client_init(self, *args, **kwargs) _install_client_auth(self) def _seamed_async_client_init(self, *args, **kwargs): async_client_init(self, *args, **kwargs) _install_client_auth(self) TestClient.__init__ = _seamed_test_client_init httpx.AsyncClient.__init__ = _seamed_async_client_init try: yield finally: TestClient.__init__ = test_client_init httpx.AsyncClient.__init__ = async_client_init @pytest.fixture(autouse=True, scope="function") def _auth_seam_optout(request): """Disable the auth seam for a ``@pytest.mark.no_auth_seam`` test.""" global _AUTH_SEAM_ON previous = _AUTH_SEAM_ON _AUTH_SEAM_ON = request.node.get_closest_marker("no_auth_seam") is None try: yield finally: _AUTH_SEAM_ON = previous # =================================================================== # Host-timezone leak guard # =================================================================== @pytest.fixture(autouse=True, scope="session") def _clear_dotenv_timezone(): """Drop a ``.env``-injected ``TZ`` once, before any test runs. litellm calls a bare ``load_dotenv()`` at import, and the bare form searches *parent* directories -- so a worktree with no ``.env`` of its own still inherits the checkout's. That publishes ``TZ`` into ``os.environ`` during collection without a matching ``time.tzset()``, leaving the variable and the C library's cached zone disagreeing. The first test to call ``tzset()`` then reconciles them mid-test, and ``_no_host_timezone_leak`` reports the change against whichever test happened to make the call. It only bites where the host zone differs from the ``.env`` one, which is why CI -- which has no ``.env`` at all -- never sees it. Session scope is load-bearing: the clear has to land outside every function-scoped fixture's window, including the guard's own. Autouse fixtures of equal scope are ordered alphabetically, so ``_no_host_timezone`` sets up first and tears down last; doing this per test would fall inside that window and read as a leak. """ if os.environ.pop("TZ", None) is not None: time.tzset() yield @pytest.fixture(autouse=True, scope="function") def _no_host_timezone_leak(): """Fail the test that leaves the process on a different host timezone. Finalizers run LIFO: a bare ``request.addfinalizer(time.tzset)`` runs *before* monkeypatch restores ``TZ``, leaving every later test on the wrong host zone. This fixture's teardown runs after monkeypatch's own, which is exactly the ordering the bug turns on. """ import os as _os import time as _time before = (_os.environ.get("TZ"), _time.tzname) yield after = (_os.environ.get("TZ"), _time.tzname) assert after == before, ( f"test leaked the host timezone: TZ/tzname was {before} before the test " f"and {after} after. Undo the monkeypatch *before* calling time.tzset() " f"so the C library re-reads the restored TZ, not the patched one." ) # =================================================================== # Working-directory guard # =================================================================== @pytest.fixture(autouse=True, scope="function") def restore_cwd(): """Restore the process working directory after every test. Leak guarded: raw ``os.chdir`` calls scattered through the suite, plus any exception path that skips their ``finally``, leave the process parked in a foreign (often deleted ``tmp_path``) directory. The compose generator resolves ``SERVICES_DIR`` relative to the cwd, so a leaked cwd silently changes what later tests render. This is a backstop, not a licence: individual ``os.chdir`` sites still own their own restore. Deliberately cheap — it snapshots the path string and nothing else. """ try: original = os.getcwd() except OSError: # Already parked in a deleted directory when this test started. original = str(_REPO_ROOT) yield try: os.chdir(original) except OSError: # The test deleted the directory it started in. os.chdir(_REPO_ROOT) # =================================================================== # Root-logging guard # =================================================================== @pytest.fixture(autouse=True, scope="function") def restore_root_logging(): """Return root logging to its pre-collection state before and after every test. Leak guarded: ``configure_logging()`` (``src/osprey/utils/logger.py``) makes three process-global changes — it sets the root logger level, installs an Osprey ``RichHandler``, and raises six third-party loggers to WARNING. Any test that reaches an entry point triggers all three, and they would then decide what ``caplog`` sees in every test that ran after it on the same worker, including whether httpx/urllib3 records are captured at all. The baseline is ``_PRISTINE_LOGGING``, captured at conftest import, rather than a per-test snapshot. A per-test snapshot cannot see far enough back: pytest sets higher-scoped fixtures up *before* this function-scoped one, so a module-scoped fixture that configures logging is already reflected in the snapshot and would be preserved rather than undone, for every later test on the worker. ``tests/cli/test_dockerfile_template.py`` runs the CLI from a module-scoped fixture and does exactly that. Only ``RichHandler`` instances are ever removed. ``caplog`` installs its ``LogCaptureHandler`` around each test, and anything a test or fixture owns itself, are left alone. The CLI adds a fourth process-global change: it installs its altitude gate (``osprey.cli.altitude``) as a filter on that ``RichHandler``, which decides what a run renders. A handler a test survives with — one that was already pristine, or one a fixture owns — would carry that gate into every later test on the worker, so any gate is stripped as well. """ root = logging.getLogger() # `in` on handlers is an identity check — they define no __eq__. pristine_handlers = _PRISTINE_LOGGING["root_handlers"] def _strip_altitude_gate(handler: logging.Handler) -> None: # A RichHandler carries no filters until the CLI gates it, so the # import — which pulls in osprey.cli — stays out of runs that never # touch the CLI. if not handler.filters: return from osprey.cli.altitude import lift_gate lift_gate(handler) def _reset() -> None: root.setLevel(_PRISTINE_LOGGING["root_level"]) for handler in list(root.handlers): if isinstance(handler, RichHandler): _strip_altitude_gate(handler) if handler not in pristine_handlers: root.removeHandler(handler) for name, level in _PRISTINE_LOGGING["third_party"].items(): logging.getLogger(name).setLevel(level) _reset() yield _reset() # =================================================================== # Health offload accounting guard # =================================================================== @pytest.fixture(autouse=True, scope="function") def reset_health_offload_state(): """Zero the health runner's abandoned-thread accounting around every test. Leak guarded: ``osprey.health.offload`` counts abandoned worker threads cumulatively for the life of the process — correct for a real run, shared state for a test suite. ``tests/health/test_offload.py`` really abandons threads, and afterwards every in-process ``osprey health`` invocation in ``tests/cli`` takes the ``os._exit`` branch (``src/osprey/cli/health_cmd.py:323-326``) instead of ``sys.exit``. Under xdist that kills the worker and everything still queued on it. Reset happens only between tests, never inside one: ``tests/health/test_offload.py`` asserts the real counter. """ from osprey.health.offload import reset_abandoned_state reset_abandoned_state() yield reset_abandoned_state() # =================================================================== # Marker-driven resource skip-gating # =================================================================== # # Tests use @pytest.mark.requires_<resource> to declare what they need. # This hook turns those markers into real skips when the resource is # unavailable. Markers are registered in pyproject.toml under # [tool.pytest.ini_options].markers; predicates live here. # # To add a new resource: append to _RESOURCE_CHECKS. Predicate must be # zero-arg, side-effect-free, and fast (gets called once per item per # resource at collection time). def _has_als_apg_api_key() -> bool: import os as _os return bool(_os.environ.get("ALS_APG_API_KEY")) def _has_anthropic_api_key() -> bool: import os as _os return bool(_os.environ.get("ANTHROPIC_API_KEY")) def _has_any_provider_api_key() -> bool: """True if any supported LLM provider key is set. Mirrors the inline detection in tests/e2e/test_llm_channel_namer.py: als-apg, cborg, amsc-i2, anthropic. """ import os as _os return any( _os.environ.get(k) for k in ("ALS_APG_API_KEY", "CBORG_API_KEY", "AMSC_I2_API_KEY", "ANTHROPIC_API_KEY") ) def _has_openai_api_key() -> bool: import os as _os return bool(_os.environ.get("OPENAI_API_KEY")) def _has_google_api_key() -> bool: import os as _os return bool(_os.environ.get("GOOGLE_API_KEY")) def _has_cborg_api_key() -> bool: import os as _os return bool(_os.environ.get("CBORG_API_KEY")) def _is_ollama_available() -> bool: """True if a local Ollama server responds at localhost:11434.""" try: import requests return requests.get("http://localhost:11434/api/tags", timeout=2).status_code == 200 except Exception: return False _RESOURCE_CHECKS: dict[str, tuple[callable, str]] = { "requires_als_apg": (_has_als_apg_api_key, "ALS_APG_API_KEY not set"), "requires_anthropic": (_has_anthropic_api_key, "ANTHROPIC_API_KEY not set"), "requires_api": ( _has_any_provider_api_key, "No LLM provider API key set (ALS_APG_API_KEY / CBORG_API_KEY / " "AMSC_I2_API_KEY / ANTHROPIC_API_KEY)", ), "requires_ollama": (_is_ollama_available, "Ollama not reachable at localhost:11434"), "requires_openai": (_has_openai_api_key, "OPENAI_API_KEY not set"), "requires_google": (_has_google_api_key, "GOOGLE_API_KEY not set"), "requires_cborg": (_has_cborg_api_key, "CBORG_API_KEY not set"), } def pytest_collection_modifyitems(config, items): """Auto-skip items whose `requires_<resource>` marker's resource is missing. Each predicate is evaluated at most once per pytest run via cache. A missing resource adds a real `pytest.mark.skip(reason=...)`; satisfied markers are no-ops. """ cache: dict[str, bool] = {} for item in items: for marker_name, (predicate, reason) in _RESOURCE_CHECKS.items(): if marker_name not in item.keywords: continue available = cache.get(marker_name) if available is None: available = predicate() cache[marker_name] = available if not available: item.add_marker(pytest.mark.skip(reason=reason)) # =================================================================== # CI diagnostics: what survives a kill # =================================================================== # # A test that FAILS reports itself. A lane that is KILLED — job timeout, runner # OOM, a wedged worker held until the step cap — reports nothing at all: the # signal lands while pytest's output is still in a buffer. These hooks keep a # continuously flushed record on disk instead, so the post-mortem has something # to read. See tests/ci_diagnostics.py for the file formats and for why # faulthandler must target a file rather than a worker's stderr. # # Entirely gated on OSPREY_CI_DIAG_DIR, which only the CI lanes set: with the # variable unset this installs nothing and costs nothing. _CI_DIAGNOSTICS: ci_diagnostics.DiagnosticsRecorder | None = None def pytest_configure(config): """Open the per-worker records and arm the stack dumper.""" # Registered here rather than in pyproject.toml so the seam and its opt-out # marker live in one file; `--strict-markers` would otherwise reject it. config.addinivalue_line( "markers", "no_auth_seam: opt this test out of the TestClient auth seam — for a " "test that asserts an unauthenticated 401/403 through a TestClient.", ) global _CI_DIAGNOSTICS _CI_DIAGNOSTICS = ci_diagnostics.recorder_from_env() if _CI_DIAGNOSTICS is not None: _CI_DIAGNOSTICS.start() def pytest_runtest_logstart(nodeid, location): """Fires before setup — so a test that never returns is still on record.""" if _CI_DIAGNOSTICS is not None: _CI_DIAGNOSTICS.record("start", nodeid=nodeid) def pytest_runtest_logfinish(nodeid, location): """Fires after teardown. A `start` left unmatched is the wedged test.""" if _CI_DIAGNOSTICS is not None: _CI_DIAGNOSTICS.record("finish", nodeid=nodeid) def pytest_unconfigure(config): if _CI_DIAGNOSTICS is not None: _CI_DIAGNOSTICS.stop() # =================================================================== # xdist distribution: file-or-group scheduling # =================================================================== # # The unit lane runs `-n 4 --dist loadgroup`. Stock LoadGroupScheduling gives # every nodeid without an `@group` suffix its own scope, so unmarked tests get # no file-level pinning at all and module-level state can straddle workers. # FileOrGroupScheduling keeps upstream's behaviour for `xdist_group`-marked # nodeids and gives everything else exact `--dist loadfile` semantics. # # `_split_scope` is an underscore method of xdist's documented scheduler # extension point; tests/infrastructure/test_xdist_scheduler.py pins both the # override's behaviour and the fact that upstream still calls it. try: from xdist.scheduler import LoadGroupScheduling as _LoadGroupScheduling except ImportError: # pragma: no cover - xdist is a dev/CI-only dependency FileOrGroupScheduling = None else: class FileOrGroupScheduling(_LoadGroupScheduling): """LoadGroupScheduling that falls back to file scope, not per-test scope.""" def _split_scope(self, nodeid: str) -> str: """Group by `@group` suffix when present, else by test file path. The `@`-after-`]` check mirrors upstream's, so a parametrize value containing `@` is not mistaken for a group suffix. """ if nodeid.rfind("@") > nodeid.rfind("]"): return super()._split_scope(nodeid) return nodeid.split("::", 1)[0] def pytest_xdist_make_scheduler(config, log): """Use FileOrGroupScheduling for `--dist loadgroup`, stock xdist otherwise. Returning None hands the choice back to xdist, leaving the e2e lane's `--dist loadfile` and ad-hoc `--dist load`/`worksteal` runs untouched. """ if FileOrGroupScheduling is None: return None if config.getoption("dist", None) != "loadgroup": return None return FileOrGroupScheduling(config, log) # =================================================================== # Prompt Testing Helpers # =================================================================== class PromptTestHelpers: """Helper methods for testing prompt structure and content.""" @staticmethod def extract_section(prompt: str, section_header: str) -> str: """Extract a specific section from the prompt by its header. Args: prompt: The full prompt text section_header: The header marking the start of the section Returns: The extracted section content (without the header) """ lines = prompt.split("\n") section_lines = [] in_section = False for line in lines: if section_header in line: in_section = True continue if in_section: # Stop at next all-caps header with colon if ( line.strip() and line.strip().replace(" ", "").replace("'", "").isupper() and ":" in line ): break section_lines.append(line) return "\n".join(section_lines).strip() @staticmethod def get_section_positions(prompt: str, *section_headers: str) -> dict[str, int]: """Get the positions of multiple section headers in the prompt. Args: prompt: The full prompt text *section_headers: Variable number of section headers to find Returns: Dictionary mapping section headers to their positions (-1 if not found) """ positions = {} for header in section_headers: try: positions[header] = prompt.index(header) except ValueError: positions[header] = -1 return positions # =================================================================== # Pytest Fixtures # =================================================================== @pytest.fixture def prompt_helpers(): """Fixture providing prompt testing helper methods.""" return PromptTestHelpers # =================================================================== # Test Configuration Helpers # =================================================================== @pytest.fixture def test_config(tmp_path): """Fixture providing a minimal test configuration file. Creates a temporary config.yml with sensible defaults for testing. Tests can override or extend this as needed. Returns: Path to the created config file Examples: Basic usage:: def test_something(test_config): # Config file exists at test_config assert test_config.exists() # Set environment to use it os.environ['CONFIG_FILE'] = str(test_config) With custom modifications:: def test_custom(test_config): # Read existing config import yaml with open(test_config) as f: config = yaml.safe_load(f) # Modify config['execution']['execution_method'] = 'subprocess' # Write back with open(test_config, 'w') as f: yaml.dump(config, f) """ import yaml config_file = tmp_path / "config.yml" # Create a test registry using extend_framework_registry helper registry_file = tmp_path / "registry.py" registry_file.write_text( """ # Test registry for integration tests - extends framework with empty additions from osprey.registry import RegistryConfigProvider, extend_framework_registry class TestRegistryProvider(RegistryConfigProvider): '''Test registry provider that extends framework defaults.''' def get_registry_config(self): # Use extend_framework_registry helper - the recommended way # This extends the framework registry with no additions return extend_framework_registry() """ ) # Minimal working configuration for tests config = { "project_root": str(tmp_path), "registry_path": str(registry_file), "approval": { "enabled": True, "default_policy": "selective", }, "control_system": { "type": "mock", # Use mock for tests - default patterns include write_channel/read_channel }, "execution": { "execution_method": "subprocess", "limits": {"max_retries": 3, "max_execution_time_seconds": 30}, }, "models": { "orchestrator": {"provider": "openai", "model_id": "gpt-4"}, "python_code_generator": {"provider": "openai", "model_id": "gpt-4"}, }, } with open(config_file, "w") as f: yaml.dump(config, f) # Do NOT initialize registry here - let each test handle registry initialization # to avoid state pollution between tests return config_file @pytest.fixture def test_config_with_approval(tmp_path): """Fixture providing a test configuration WITH approval enabled. This is specifically for testing approval workflows. """ import yaml config_file = tmp_path / "config.yml" # Create a test registry using extend_framework_registry helper registry_file = tmp_path / "registry.py" registry_file.write_text( """ # Test registry for integration tests - extends framework with empty additions from osprey.registry import RegistryConfigProvider, extend_framework_registry class TestRegistryProvider(RegistryConfigProvider): '''Test registry provider that extends framework defaults.''' def get_registry_config(self): # Use extend_framework_registry helper - the recommended way # This extends the framework registry with no additions return extend_framework_registry() """ ) # Configuration with approval ENABLED config = { "project_root": str(tmp_path), "registry_path": str(registry_file), "approval": { "enabled": True, "default_policy": "selective", }, "control_system": { "type": "mock", # Use mock for tests - default patterns include write_channel/read_channel }, "execution": { "execution_method": "subprocess", "limits": {"max_retries": 3, "max_execution_time_seconds": 30}, }, "models": { "orchestrator": {"provider": "openai", "model_id": "gpt-4"}, "python_code_generator": {"provider": "openai", "model_id": "gpt-4"}, }, } with open(config_file, "w") as f: yaml.dump(config, f) # Do NOT initialize registry here - let the test do it # to avoid state pollution between tests return config_file @pytest.fixture(scope="session") def graphdb_plugin_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: """A directory holding neosemantics + APOC, resolved once per session. Every lane that starts a real graph store mounts the same two jars, and the expensive half of resolving them is a release download — so this is shared across lanes even though the *stores* are not (two of them wipe the graph between corpora and therefore run their own module-scoped container). Skips, with the reason, on a host that could not start the container: see :func:`tests._graphdb_container.resolve_plugin_dir`. """ from tests._graphdb_container import resolve_plugin_dir return resolve_plugin_dir(tmp_path_factory) # =================================================================== # Control context — the record and the per-server reports # =================================================================== # # Re-exported from ``tests._control_context_fixtures`` so that every suite # downstream of the control target writes the two files the same way, and a # schema change fails all of them at once instead of one at a time. The # writers are plain functions there for suites that would rather import them # at module level; these are the factory fixtures that hand the same functions # to a test that asks for them by name. @pytest.fixture def write_control_context(): """Write a control-context record under an explicit agent-data root. ``write_control_context(root, target="va", generation=3, posture={"va": "sandbox"})`` — see :func:`tests._control_context_fixtures.write_control_context` for the owner and ``last_switch`` arguments. Defaults to a record this process owns, so a test gets a live owner without asking for one. """ from tests._control_context_fixtures import write_control_context as _write return _write @pytest.fixture def write_server_report(): """Write one controls server's report under an explicit agent-data root. ``write_server_report(root, 4321, session="s", applied_target="va", applied_generation=3)`` — see :func:`tests._control_context_fixtures.write_server_report`. Every field but the PID defaults to "this server has not answered yet". """ from tests._control_context_fixtures import write_server_report as _write return _write @pytest.fixture def control_context_root(tmp_path, monkeypatch): """A scratch agent-data root, stamped, with the reader cache dropped. The three lines every control-context suite opens with: a root that exists, ``OSPREY_AGENT_DATA_ROOT`` pointing at it, and no record parsed from a previous test still held in the module cache. The ``control_target/`` directory is created, so a test can write a raw file into it without reaching for ``mkdir`` first. """ from osprey_connectors import control_context, posture_store root = tmp_path / "agent_data" (root / posture_store.STATE_DIR_NAME).mkdir(parents=True, exist_ok=True) monkeypatch.setenv(posture_store.AGENT_DATA_ROOT_ENV_VAR, str(root)) control_context.invalidate_cache() yield root control_context.invalidate_cache() # =================================================================== # Graph search index # =================================================================== @dataclass(slots=True) class GraphIndexCache: """Every graph search index this session has built, keyed by corpus digest. Held for the whole session because the thing being cached is not a render's file but the *parse* behind it: two builds of the same corpus, in two tests, in two directories, derive byte-identical rows from byte-identical text. """ directory: Path """Where the cached indexes live — one ``<digest>.duckdb`` per corpus.""" entries: dict[str, Path] = field(default_factory=dict) """Digest of the corpus TEXT -> the index built from it, inside :attr:`directory`.""" @pytest.fixture(scope="session") def graph_index_cache(tmp_path_factory: pytest.TempPathFactory) -> GraphIndexCache: """The session's built indexes, shared by every render the suite performs. Requested by the autouse stub below, and by the tests that assert on the caching itself: a test that needs a corpus to be built for real can drop its entry here first, and get a deterministic builder count whatever ran before it. """ return GraphIndexCache(directory=tmp_path_factory.mktemp("graph-index-cache")) @pytest.fixture(autouse=True, scope="function") def _stub_graph_index_builds(request, monkeypatch, graph_index_cache: GraphIndexCache): """Build each corpus the suite renders once per session, not once per render. ``osprey build`` derives a graph-mode project's channel search index from the Turtle corpus the render staged, which is an rdflib parse of the whole file — about a second and a half for the corpus the control-assistant preset ships. The suite renders that preset over and over, in the build tests, the deployment tests and every exemplar render, and each of those renders parses the identical bytes again. So the parse is memoized across the whole session on the digest of the corpus TEXT, the same key the build's own per-build memo uses (:attr:`osprey.cli.build_cmd._SharedRenderInputs.graph_indexes`), and for the same reason: every render re-copies the profile's ``data/`` tree, so the staged corpus is a different file with a fresh mtime every time and a path or stat key would miss on every pass. What is stubbed is only :func:`osprey.cli.build_cmd._build_graph_index` — the step that turns a corpus into a file. ``_graph_index_target`` still resolves the corpus and the index path out of the rendered config, and the index that lands in the render is the real builder's output for that exact corpus, so a test can still read it, query it, or hand it to the roster. A corpus that cannot be read or parsed is handed straight to the real hook, which owns the warning and the decision that a failure costs the index rather than the build — one copy of that fact, not two. Opt out with ``@pytest.mark.real_graph_index`` when the derivation itself is what a test is about. """ if request.node.get_closest_marker("real_graph_index"): return import shutil from osprey.cli import build_cmd real_hook = build_cmd._build_graph_index def _cached_build_graph_index(shared, target, progress): # Imported here, at call time, for two reasons: the fixture stays # import-light for the tests that never render anything, and a test # that counts builder calls by patching the package attribute still # sees this path go through it. from osprey.services.channel_finder.graph_index import build_graph_index from osprey.services.facility_knowledge.seeder.graph_seeder import ttl_sha256 try: digest = ttl_sha256(target.corpus_path.read_text(encoding="utf-8")) cached = graph_index_cache.entries.get(digest) if cached is None: cached = graph_index_cache.directory / f"{digest}.duckdb" build_graph_index(target.corpus_path, cached) if not cached.is_file(): raise FileNotFoundError(cached) # Recorded only once the file exists, so a build that wrote # nothing cannot poison every later render of this corpus for # the rest of the session. graph_index_cache.entries[digest] = cached target.index_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(cached, target.index_path) except Exception: # noqa: BLE001 - the real hook decides what a failure costs return real_hook(shared, target, progress) # The build's own memo, kept consistent: a later render pass of the # same build reads it before asking for an index at all. shared.graph_indexes.setdefault(digest, target.index_path) progress( " ✓ Reused the session-cached channel search index built from %s", target.corpus_path.name, ) return target.index_path #: Marks this as the stub, so a test can assert which hook it is running #: against without inspecting where the function was defined. _cached_build_graph_index.stubbed_by_conftest = True monkeypatch.setattr(build_cmd, "_build_graph_index", _cached_build_graph_index)
"""
Pytest configuration and shared test utilities.
This module provides shared fixtures and utilities for all Osprey tests.
"""
import logging
import os
import time
from dataclasses import dataclass, field
from pathlib import Path
import pytest
from rich.logging import RichHandler
from osprey.utils.logger import QUIET_THIRD_PARTY_LOGGERS
from tests import ci_diagnostics
#: Repo root — the fallback when a test leaves the process in a deleted cwd.
_REPO_ROOT = Path(__file__).resolve().parent.parent
#: Root-logging state as it was before collection — see ``restore_root_logging``.
#: Captured at conftest import, which is the only moment guaranteed to precede
#: every test *and* every higher-scoped fixture.
_PRISTINE_LOGGING = {
"root_level": logging.getLogger().level,
"root_handlers": list(logging.getLogger().handlers),
"third_party": {name: logging.getLogger(name).level for name in QUIET_THIRD_PARTY_LOGGERS},
}
# ===================================================================
# Collection-time environment pollution guard
# ===================================================================
#
# Importing a test module can import third-party code that loads a .env at
# import time — litellm calls dotenv.load_dotenv() on import, and python-dotenv
# walks UP from its package directory, so on a developer machine it can find an
# ancestor checkout's .env and inject real credentials, PROJECT_ROOT, and TZ
# into os.environ before the first test runs. CI has no ancestor .env, so
# whatever those variables change locally is invisible there.
#
# Only that injection is undone. A blanket restore would also strip the
# coordination variables some test modules legitimately export at import time
# (tests/va/* set EPICS_CA_SERVER_PORT et al. so the soft-IOC subprocesses they
# spawn share their ports), so a key is removed only when it BOTH appeared
# during collection AND carries exactly the value an ancestor .env defines.
# import-time required because `osprey.cli.styles` builds its module-level
# Rich Console at import time and Rich reads FORCE_COLOR in `Console.__init__`,
# so a fixture runs too late to un-force a console that already exists: the
# color-forcing variables must be gone BEFORE collection imports any osprey
# module. A developer terminal that
# exports FORCE_COLOR otherwise turns CliRunner captures into ANSI-laced text
# and fails seven CLI tests on pristine main — CI is green only because its
# runners never export it. `NO_COLOR` is left alone: unset is the same default
# CI runs under.
os.environ.pop("FORCE_COLOR", None)
os.environ.pop("CLICOLOR_FORCE", None)
# Rich deliberately clamps ``TERM=dumb`` consoles to 80 columns and suppresses
# cursor control even when a test explicitly requests ``force_terminal=True``.
# CI presents a capable TERM, so normalize only the dumb/unknown host values;
# tests for a restricted terminal can still set one explicitly after collection.
if os.environ.get("TERM", "").lower() in {"dumb", "unknown"}:
os.environ["TERM"] = "xterm-256color"
_PRE_COLLECTION_ENV = dict(os.environ)
def _ancestor_dotenv_values() -> dict[str, str]:
"""Every assignment an ancestor .env could have injected, keyed by var."""
from dotenv import dotenv_values
injected: dict[str, str] = {}
for directory in (_REPO_ROOT, *_REPO_ROOT.parents):
candidate = directory / ".env"
if candidate.is_file():
injected.update({k: v for k, v in dotenv_values(candidate).items() if v is not None})
return injected
def pytest_collection_finish(session):
"""Undo ancestor-.env injection performed by imports during collection."""
import time
added = set(os.environ) - set(_PRE_COLLECTION_ENV)
if added:
injected = _ancestor_dotenv_values()
for key in added:
if key in injected and os.environ[key] == injected[key]:
del os.environ[key]
# Re-sync the C library's cached zone with the (possibly restored) TZ.
# Collection can leave the two disagreeing (TZ injected, cache not), and
# then the first test to call tzset() flips tzname mid-suite and reads as
# a leaker to _no_host_timezone_leak below.
time.tzset()
# ===================================================================
# Environment guard

Xet Storage Details

Size:
51.8 kB
·
Xet hash:
1b5efc16d3a4fa9bacf5b18dc4c46a5586b9cb0cb490760933eabbf52e44ec4a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.