Buckets:

download
raw
41.9 kB
"""
Global test configuration for pytest.
This file is automatically loaded by pytest and contains setup that applies to all tests.
"""
# CRITICAL: Set import protection BEFORE any other imports
import faulthandler
import os
import signal
import sys
import tempfile
import threading
import time
# #956 DIAGNOSTIC — make a hang self-report instead of dying silently.
#
# The intermittent pytest-xdist exit-hang was root-caused (non-daemon FFI
# threads in setup/attestation.py that leaked past their join timeout), but a
# leak that recurs from a NEW source must not cost another 30-minute silent
# timeout to diagnose. faulthandler here dumps EVERY thread's stack — naming the
# blocked frame's file:line — in two situations the sessionfinish guard below
# cannot cover on its own:
# * on a fatal signal (segfault etc.), and
# * on SIGTERM, which is exactly how CI kills the hang: the shard wraps pytest
# in `timeout --signal=SIGTERM ... 1800`, so registering here converts the
# 30-min silent death into a full per-thread traceback for whichever process
# received the signal (at minimum the controller).
# Pure diagnostics: fires only on an actual signal, never during a healthy run.
faulthandler.enable()
if hasattr(faulthandler, "register") and hasattr(signal, "SIGTERM"):
try:
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
except (ValueError, OSError):
# Some environments disallow registering handlers (e.g. non-main thread
# at import). Diagnostics are best-effort; never break collection.
pass
# WATCHDOG DUMP — the SIGTERM handler above proved insufficient: when CI's
# job-level timeout-minutes fires, GitHub tears the process down fast enough
# that a signal-driven dump does not reach the captured log. This is
# signal-independent: a dedicated timer thread dumps EVERY thread's stack after
# a fixed wall-clock interval, straight to fd 2, so a hung shard self-reports
# INTO THE LIVE LOG well before it is killed.
#
# 1440s (24 min): healthy test shards finish in ~17-19 min and exit — the timer
# dies with the process, no dump. A hung shard (killed at the 30-min job
# timeout) gets its stacks dumped at 24 min, 6 min before death, while the step
# log is still being tailed. Chosen to sit ABOVE the slowest healthy shard and
# BELOW the kill. repeat=True in case the first dump races teardown.
#
# This is how we find the residual leak the attestation daemonization did NOT
# fix (a non-thread mechanism — leaked child process holding the execnet fd, or
# a C-extension thread) — it will name the exact blocked frame next hang.
#
# MEASURED 2026-08-02: `faulthandler.dump_traceback_later()` DOES NOT SURVIVE A
# PYTEST RUN. Set the interval to 3s and run a 9s suite and you get zero dumps —
# pytest's own faulthandler plugin calls `faulthandler.enable(file=<dup'd fd>)`
# at configure time and the pending timer is lost. So this watchdog never fired,
# which is why the 2026-08-02 shard-3 hang produced no stacks at all despite
# being armed for exactly that.
#
# Replaced with a plain daemon timer thread calling `dump_traceback()` directly.
# It is ours, nothing else touches it, and it is a daemon so it cannot itself
# hold the process open.
# A dup of the REAL stderr, taken at import before pytest installs capture.
# `sys.stderr` is a capture buffer for most of a run, so anything a watchdog
# writes there lands in a buffer that a hung or killed process never flushes —
# which is the second reason the previous watchdog produced nothing. pytest's
# own faulthandler plugin dups the fd for exactly this reason.
try:
_REAL_STDERR_FD = os.dup(2)
except OSError: # pragma: no cover
_REAL_STDERR_FD = 2
def _emit_stacks(header: str, tag: str = "hang") -> None:
"""Write a header + every thread's stack to a FILE, and try stderr too.
THE FILE IS THE RELIABLE CHANNEL, and that is not a preference — pytest
installs fd-level capture (``--capture=fd`` is the default) BEFORE conftest
is imported, so even a dup of fd 2 taken here is a dup of pytest's capture
pipe rather than the terminal. Anything written there lands in a buffer that
a hung or SIGKILLed process never flushes. Measured: a 3s watchdog on a 9s
suite produced no visible output through any stderr route.
So write a file first and unconditionally. CI uploads it as an artifact,
which also survives the step log being truncated at the tail — exactly what
happened to the 2026-08-02 shard-3 hang, where nothing at all was captured.
The stderr attempt is kept as a bonus for `-s` / local runs.
"""
path = os.environ.get("CIRIS_TEST_HANG_DUMP_DIR", ".")
role = os.environ.get("PYTEST_XDIST_WORKER", "controller")
try:
os.makedirs(path, exist_ok=True)
with open(os.path.join(path, f"{tag}-stacks-{role}.txt"), "a", encoding="utf-8") as fh:
fh.write(header)
fh.write(f"# live_threads={threading.active_count()}\n")
for t in threading.enumerate():
fh.write(f"# {t.name} daemon={t.daemon} type={type(t).__name__}\n")
fh.flush()
faulthandler.dump_traceback(file=fh, all_threads=True)
fh.flush()
os.fsync(fh.fileno())
except Exception: # noqa: BLE001 — diagnostics must never break the run
pass
try:
os.write(_REAL_STDERR_FD, header.encode("utf-8", "replace"))
with os.fdopen(os.dup(_REAL_STDERR_FD), "w", closefd=True) as fh:
faulthandler.dump_traceback(file=fh, all_threads=True)
fh.flush()
except Exception: # noqa: BLE001
pass
_HANG_WATCHDOG_SECONDS = int(os.environ.get("CIRIS_TEST_HANG_WATCHDOG_SECONDS", "1440"))
if _HANG_WATCHDOG_SECONDS > 0:
def _hang_watchdog() -> None:
while True:
time.sleep(_HANG_WATCHDOG_SECONDS)
_emit_stacks(
f"\n[conftest #956] WATCHDOG: still running after "
f"{_HANG_WATCHDOG_SECONDS}s — every thread's stack follows.\n"
)
try:
threading.Thread(target=_hang_watchdog, name="ciris-hang-watchdog", daemon=True).start()
except Exception: # noqa: BLE001
pass
# CONTROLLER DEADLINE — the bound the 2026-08-02 main-branch hang proved missing.
#
# Run 30754955268, shard 2: gw1/2/3 finished, hit the sessionfinish deadline
# below, and self-exited — that mitigation WORKED. gw0 wedged inside a single
# test (`test_database_maintenance_multi_occurrence.py::TestAdapterConfigHelpers::
# test_find_adapter_ids_from_configs_ignores_non_type_keys`, [TEST_START] with no
# result) at native level: pytest-timeout's 60s thread-method timer never got the
# GIL, which points at the Rust persist/sqlx teardown race documented in
# tests/fixtures/database.py:12. The controller then sat in
# `dsession.py loop_once -> queue.get()` for 9.5 minutes until the EXTERNAL
# `timeout` SIGTERM'd it at 1500s (exit 124, no Python stacks from the bound
# that killed it).
#
# Why none of the existing bounds could fire:
# * `--session-timeout` is evaluated inside pytest-timeout's runtest_protocol
# hookwrapper — WORKERS ONLY, and only between items. Once the last live
# worker is wedged *inside* an item, nothing evaluates it ever again. It
# bounds the loop only while workers are still completing items.
# * `_arm_exit_deadline` (below) arms at pytest_sessionfinish — the controller
# never got there. Its dump showed no ciris-exit-deadline thread.
# * pytest-timeout's per-test timer needs the GIL; a native block starves it.
#
# This deadline is wall-clock, armed at import in the CONTROLLER ONLY, and
# OPT-IN VIA ENV (default off): a healthy local full-suite run can legitimately
# exceed any number we would pick here, so the number belongs to the CI job that
# knows its own budget. CI sets it just below the external SIGTERM so the
# process self-reports with stacks and a real exit code instead of dying
# silently 2 minutes later.
_CONTROLLER_DEADLINE_SECONDS = int(os.environ.get("CIRIS_TEST_CONTROLLER_DEADLINE_SECONDS", "0"))
if _CONTROLLER_DEADLINE_SECONDS > 0 and "PYTEST_XDIST_WORKER" not in os.environ:
def _controller_deadline() -> None:
time.sleep(_CONTROLLER_DEADLINE_SECONDS)
_emit_stacks(
f"\n[conftest #956] CONTROLLER DEADLINE: session still running after "
f"{_CONTROLLER_DEADLINE_SECONDS}s — a worker is likely wedged in native code. "
f"Dumping stacks and exiting 1 so the bound that killed the run is the one "
f"that reported it.\n"
)
os._exit(1)
try:
threading.Thread(target=_controller_deadline, name="ciris-controller-deadline", daemon=True).start()
except Exception: # noqa: BLE001
pass
# BELT for the residual #956 leak: the CIRISVerify ThreadPoolExecutor.
#
# concurrent.futures registers `_python_exit` via atexit to JOIN every executor
# worker thread at interpreter shutdown (executor workers are non-daemon since
# 3.9). ciris_verify/ffi_bindings/client.py holds a ThreadPoolExecutor and
# submits CIRISVerify FFI calls to it. In CI there is no attestation hardware,
# so an FFI task can block indefinitely — and `shutdown(wait=False)` in the
# client's __del__ cannot cancel a task already running in a worker. That leaves
# a non-daemon worker stuck in the FFI, and `_python_exit` blocks joining it
# FOREVER at shutdown. It is invisible to the threading.Thread AST lint (the
# threads are stdlib-created) and it runs in the atexit phase, AFTER the
# sessionfinish guard, which is why the guard did not catch it.
#
# The test process does not need in-flight executor results at exit, so drop the
# join. The sessionfinish guard still force-exits on any lingering non-daemon
# thread; this removes the one blocking phase that runs past it. Belt only — the
# watchdog above still fires if a DIFFERENT residual mechanism remains, so this
# cannot mask a further leak.
try:
import atexit as _atexit
import concurrent.futures.thread as _cf_thread
_atexit.unregister(_cf_thread._python_exit)
except Exception: # noqa: BLE001 — best-effort; never break collection
pass
os.environ["CIRIS_IMPORT_MODE"] = "true"
os.environ["CIRIS_MOCK_LLM"] = "true"
os.environ["CIRIS_TESTING_MODE"] = "true" # Enable fallback admin credentials for tests
# Unit tests never boot the real node fold (ciris-server serve on 4243): since
# 0.5.113 the fold is fail-fast (node-fails => agent-fails) and requires a live
# embedded edge (init_edge_runtime) + SQLite engine, which bare unit tests do
# not stand up. Staged QA / integration boots override this explicitly.
os.environ.setdefault("CIRIS_NODE_FOLD", "false")
# Pin language to English so handler-content assertions stay stable regardless
# of the developer's local CIRIS_PREFERRED_LANGUAGE in .env. Tests that
# intentionally exercise localization can override via monkeypatch.
os.environ["CIRIS_PREFERRED_LANGUAGE"] = "en"
# CRITICAL: Override log directory for tests to prevent container interference
# Tests should NEVER write to the main logs directory that containers use
os.environ["CIRIS_LOG_DIR"] = "test_logs"
os.environ["CIRIS_DATA_DIR"] = "test_data"
# PERFORMANCE: Use tmpfs for all temp files to reduce disk I/O during tests
# /dev/shm is a RAM-based tmpfs available on most Linux systems
#
# Per-worker isolation (added 2026-04-30 for the -n 28 parallel-execution
# regression in db-maintenance-multi-occurrence + dsar-multi-source):
# every worker gets its OWN TMPDIR subdir under _TMPFS_BASE. Without this,
# all workers shared `/dev/shm/ciris_tests/` and pytest-xdist's session-
# level cleanup of `pytest-of-emoore/pytest-0/popen-gw{N}/` could delete
# one worker's tmpdir while another worker was still using it (FileNotFoundError
# on iterdir during test setup, observed in -n 28 runs).
_TMPFS_BASE = "/dev/shm/ciris_tests"
if os.path.isdir("/dev/shm"):
import shutil
is_xdist_worker = os.environ.get("PYTEST_XDIST_WORKER") is not None
# INVARIANT: no process deletes a directory it does not exclusively own.
#
# The previous version gave the non-worker process `_TMPFS_BASE` ITSELF and
# rmtree'd it. Because every worker's dir is a CHILD of that base, any
# process importing this conftest without PYTEST_XDIST_WORKER set — the
# xdist controller included — destroyed every live worker's TMPDIR
# mid-session. Workers then failed in `tempfile.mkdtemp()` itself, so
# switching a test away from `tmp_path` did NOT help; the failure counts
# varied run to run (302-412 errors) and made every `-n` result unreadable.
# See #947.
if is_xdist_worker:
worker_id = os.environ["PYTEST_XDIST_WORKER"] # e.g. "gw23"
_TMPFS_DIR = os.path.join(_TMPFS_BASE, worker_id)
else:
# The controller/serial process gets its OWN subdir, a sibling of the
# workers' rather than their parent.
_TMPFS_DIR = os.path.join(_TMPFS_BASE, "main")
# Stale-sweep, not an eager wipe: the stated goal is keeping /dev/shm from
# filling up over time, which an age check satisfies without touching
# anything a concurrent run is using. Only the base's direct children are
# considered, and only when nothing has written to them for an hour.
if not is_xdist_worker and os.path.isdir(_TMPFS_BASE):
_stale_cutoff = time.time() - 3600
for _entry in os.listdir(_TMPFS_BASE):
_path = os.path.join(_TMPFS_BASE, _entry)
try:
if os.path.getmtime(_path) < _stale_cutoff:
shutil.rmtree(_path, ignore_errors=True)
except OSError:
pass # racing another run's cleanup is fine; it owns that dir
os.makedirs(_TMPFS_DIR, exist_ok=True)
os.environ["TMPDIR"] = _TMPFS_DIR
tempfile.tempdir = _TMPFS_DIR
# PERFORMANCE: Disable network-dependent services in tests
# This prevents NTP checks, billing API calls, etc.
os.environ["CIRIS_DISABLE_NETWORK"] = "true"
# NOTE: Network calls are disabled via CIRIS_IMPORT_MODE and CIRIS_MOCK_LLM
# Code should check these flags and skip network operations entirely
from pathlib import Path # noqa: E402
import pytest # noqa: E402
# Load environment variables from .env file for all tests
try:
from dotenv import load_dotenv
# Find the .env file in the project root
project_root = Path(__file__).parent.parent
env_file = project_root / ".env"
if env_file.exists():
load_dotenv(env_file)
# Also load test-specific config if available
test_env_file = Path(__file__).parent / "test_config.env"
if test_env_file.exists():
load_dotenv(test_env_file, override=True)
# Also set as environment variables for non-dotenv aware code
import subprocess
result = subprocess.run(["bash", "-c", f"source {test_env_file} && env"], capture_output=True, text=True)
if result.returncode == 0:
for line in result.stdout.splitlines():
if "=" in line and line.startswith("CIRIS_"):
key, value = line.split("=", 1)
os.environ[key] = value
except ImportError:
# If python-dotenv is not installed, silently continue
pass
import gc # noqa: E402
# Import database fixtures and API fixtures - must be imported after os.environ setup
from tests.fixtures.api import random_api_port # noqa: E402
from tests.fixtures.database import ( # noqa: E402
_release_persist_engine,
clean_db,
test_db,
)
from tests.fixtures.mocks import MockRuntime # noqa: E402
from tests.fixtures.persist_engine import persist_engine # noqa: E402, F401
from tests.fixtures.runtime_control import ( # noqa: E402
mock_api_runtime_control_service,
mock_step_result_gather_context,
mock_step_result_perform_dmas,
single_step_control_response,
)
@pytest.fixture
def mock_runtime(mock_step_result_perform_dmas):
"""Create a centralized MockRuntime with step result data."""
runtime = MockRuntime()
# Connect the centralized step result fixture
runtime.pipeline_controller.get_latest_step_result.return_value = mock_step_result_perform_dmas
return runtime
# Make fixtures available to all tests by explicitly referencing them
__all__ = [
"test_db",
"clean_db",
"random_api_port",
"single_step_control_response",
"mock_step_result_perform_dmas",
"mock_step_result_gather_context",
"mock_api_runtime_control_service",
"mock_runtime",
]
@pytest.fixture(scope="session", autouse=True)
def manage_import_protection():
"""Manage import protection for the entire test session."""
# Import protection is already set at module level
# But we ensure it stays set during test collection
os.environ["CIRIS_IMPORT_MODE"] = "true"
os.environ["CIRIS_MOCK_LLM"] = "true"
yield
# After all tests are done, we can clear the protection
# (though it doesn't really matter since process is ending)
os.environ.pop("CIRIS_IMPORT_MODE", None)
@pytest.fixture(autouse=True)
def _isolate_process_global_env(monkeypatch):
"""Restore global, process-wide env vars to known values for every test.
Some tests intentionally flip `CIRIS_PREFERRED_LANGUAGE` or
`CIRIS_LLM_REPLICAS` via raw `os.environ[...] = ...` to exercise
behavior that depends on those globals. Those env vars work fine for
personal-agent deployments (one process, one operator) but they are
the wrong shape for a test harness: under `pytest -n`, xdist workers
run tests serially per worker, so a leaked mutation from test A can
change the behavior of unrelated test B and produce flaky CI
failures (e.g. a handler follow-up rendered in Amharic, or a second
LLM replica being initialized when the test expects one).
Using monkeypatch here captures the current values before each test
and restores them at teardown — even if a test mutates them with raw
`os.environ[...]` and forgets to restore, or crashes mid-test. Tests
that want a different value should still prefer `monkeypatch.setenv`
inside the test so the change is scoped and obvious.
"""
monkeypatch.setenv("CIRIS_PREFERRED_LANGUAGE", "en")
monkeypatch.delenv("CIRIS_LLM_REPLICAS", raising=False)
@pytest.hookimpl(tryfirst=True)
def pytest_runtest_setup(item): # noqa: ARG001
"""Reset the process-wide persist Engine before every test.
ciris-persist's Engine is a process-singleton: a second construction
with a different DSN raises EngineConfigMismatch, and the test suite
has ~20+ fixtures that each bootstrap a per-test engine.
This is a hook, not an autouse fixture, deliberately: a `tryfirst`
`pytest_runtest_setup` runs strictly BEFORE all fixture setup —
including class-level autouse fixtures that wire their own engine —
so no fixture-ordering race can leave a prior test's engine pinned.
`_release_persist_engine` calls ciris-persist's `reset_engine()`
(CIRISPersist#88), which un-pins the singleton handle-free — so even
an orphan engine (global nulled without close) is recovered.
"""
_release_persist_engine()
@pytest.hookimpl(trylast=True)
def pytest_runtest_teardown(item): # noqa: ARG001
"""Release the persist Engine after every test (mirror of setup)."""
_release_persist_engine()
@pytest.fixture(autouse=True, scope="function")
def cleanup_after_test(request):
"""
Ensure proper cleanup after each test.
This helps prevent interference between tests, especially when Discord is involved.
"""
yield
# Force garbage collection to clean up any lingering objects
gc.collect()
# Only add socket cleanup delay for tests that actually use network/Discord
# This saves ~0.1s per test for the majority of tests that don't need it
test_path = str(request.fspath) if hasattr(request, "fspath") else ""
needs_socket_cleanup = (
"discord" in test_path.lower()
or request.node.get_closest_marker("requires_discord_token")
or request.node.get_closest_marker("needs_socket_cleanup")
)
if needs_socket_cleanup:
time.sleep(0.1)
@pytest.fixture(autouse=True, scope="function")
def strip_leaked_incident_handlers():
"""Remove any IncidentCaptureHandler left on a process-global logger after a
test, so it cannot leak onto the next test sharing the xdist worker process.
Root cause of the recurring 'incident_handler_injection' / logging-pollution
flake (git: e8a16d764, 3f254498f): ``add_incident_capture_handler()`` does
``target_logger.addHandler(handler)`` on a process-global logger, and not
every test removes it. Under pytest-xdist the leaked handler then shows up on
the next test's logger — e.g. ``assert len(specific_logger.handlers) == 0``
sees 2. Stripping the leak at teardown fixes it at the source instead of
per-test ``xdist_group`` whack-a-mole. Surgical: only the IncidentCaptureHandler
subclass is removed, so ``caplog`` and other handlers are left untouched.
"""
yield
import logging as _logging
try:
from ciris_engine.logic.utils.incident_capture_handler import IncidentCaptureHandler
except Exception: # pragma: no cover - module always importable in-tree
return
loggers = [_logging.getLogger()]
loggers += [_logging.getLogger(name) for name in list(_logging.Logger.manager.loggerDict)]
for lg in loggers:
handlers = getattr(lg, "handlers", None)
if not handlers:
continue
kept = [h for h in handlers if not isinstance(h, IncidentCaptureHandler)]
if len(kept) != len(handlers):
lg.handlers = kept
@pytest.fixture(autouse=True, scope="function")
def clear_asyncio_module_state():
"""
Clear module-level asyncio state to prevent cross-test contamination.
This fixes the "asyncio.Event/Queue object is bound to a different event loop" error
that occurs when pytest-xdist runs tests in parallel with different event loops.
Affected modules:
1. step_decorators:
- _paused_thoughts: Dict[str, asyncio.Event] - Events bound to specific event loops
- _single_step_mode: bool - Global flag for single-step debugging
2. step_streaming:
- reasoning_event_stream._subscribers: set[asyncio.Queue] - Queues bound to event loops
- reasoning_event_stream._sequence_number: int - Counter that should reset
- reasoning_event_stream._recent_events: list - Buffer that should clear
Without this cleanup, asyncio primitives created in one test's event loop persist
and cause RuntimeError when accessed from a different test's event loop.
"""
def _clear_state():
# Clear step_decorators state
try:
from ciris_engine.logic.processors.core import step_decorators
step_decorators._paused_thoughts.clear()
step_decorators._single_step_mode = False
except ImportError:
pass # Module not yet available during collection
# Clear step_streaming singleton state
try:
from ciris_engine.logic.infrastructure.step_streaming import reasoning_event_stream
reasoning_event_stream._subscribers.clear()
reasoning_event_stream._sequence_number = 0
reasoning_event_stream._recent_events.clear()
except ImportError:
pass # Module not yet available during collection
# Clear before test
_clear_state()
yield
# Clear after test to ensure clean state for next test
_clear_state()
@pytest.fixture(autouse=True, scope="function")
def isolate_test_env_vars():
"""
Isolate environment variables that can cause test pollution.
This fixture saves any existing env vars before the test and restores
them after, ensuring parallel test runs don't interfere with each other.
"""
# Env vars that need isolation
env_vars = [
# Accord metrics (current names)
"CIRIS_ACCORD_METRICS_CONSENT",
"CIRIS_ACCORD_METRICS_CONSENT_TIMESTAMP",
"CIRIS_ACCORD_METRICS_TRACE_LEVEL",
"CIRIS_ACCORD_METRICS_FLUSH_INTERVAL",
"CIRIS_ACCORD_METRICS_ENDPOINT",
# Legacy covenant metrics (for backward compat)
"CIRIS_COVENANT_METRICS_CONSENT",
"CIRIS_COVENANT_METRICS_CONSENT_TIMESTAMP",
"CIRIS_COVENANT_METRICS_TRACE_LEVEL",
"CIRIS_COVENANT_METRICS_FLUSH_INTERVAL",
"CIRIS_COVENANT_METRICS_ENDPOINT",
# LLM provider detection (affects which provider is selected)
"CIRIS_LLM_PROVIDER",
"LLM_PROVIDER",
"OPENAI_API_KEY",
"OPENAI_API_BASE",
"ANTHROPIC_API_KEY",
"GOOGLE_API_KEY",
"GEMINI_API_KEY",
# Dual LLM service
"CIRIS_OPENAI_API_KEY_2",
"CIRIS_OPENAI_API_BASE_2",
"CIRIS_OPENAI_MODEL_NAME_2",
]
# Save existing values
saved = {var: os.environ.get(var) for var in env_vars}
# Clear them before test
for var in env_vars:
os.environ.pop(var, None)
yield
# Restore original values after test
for var, value in saved.items():
if value is not None:
os.environ[var] = value
else:
os.environ.pop(var, None)
# Import WA test harness fixtures (when available)
try:
from ciris_engine.logic.services.test_wa_auth_harness import wa_test_env, wa_test_harness, wa_test_keys
except ImportError:
# WA test harness not available yet
wa_test_harness = None
wa_test_env = None
wa_test_keys = None
# Skip Discord tests if no token is set
def pytest_configure(config):
config.addinivalue_line("markers", "requires_discord_token: mark test as requiring Discord token")
config.addinivalue_line("markers", "needs_socket_cleanup: mark test as needing socket cleanup delay")
config.addinivalue_line(
"markers",
"real_attestation: opt out of the autouse stub that no-ops "
"AuthenticationService.run_startup_attestation / _attestation_refresh_loop",
)
# =============================================================================
# Test Start/End Markers for Debugging Hanging Tests
# =============================================================================
# These hooks print markers that can be grep'd to find tests that started but
# never finished (hanging tests). Look for [TEST_START] without matching [TEST_END].
def pytest_runtest_logstart(nodeid, location):
"""Print marker when test starts - helps identify hanging tests in CI logs."""
import sys
# Flush to ensure marker appears immediately in CI logs
print(f"\n[TEST_START] {nodeid}", file=sys.stderr, flush=True)
def pytest_runtest_logfinish(nodeid, location):
"""Print marker when test finishes - pair with TEST_START to find hangs."""
import sys
print(f"[TEST_END] {nodeid}", file=sys.stderr, flush=True)
# ============================================================================
# CASCADING FAILURE PROTECTION
# ============================================================================
# These hooks and fixtures prevent a single test crash from causing
# cascading failures (e.g., 18-minute hangs due to orphan processes).
@pytest.hookimpl(tryfirst=True)
def pytest_runtest_teardown(item, nextitem):
"""Clean up any orphan async tasks after each test.
This prevents tests from leaving behind background tasks that can
cause worker crashes or hangs in pytest-xdist.
"""
import asyncio
try:
# Only cancel tasks if there's a running event loop
# get_event_loop() without a running loop is deprecated in Python 3.10+
# and there are no tasks to cancel anyway if there's no loop
try:
loop = asyncio.get_running_loop()
# Cancel any pending tasks from this test
pending = asyncio.all_tasks(loop)
for task in pending:
if not task.done():
task.cancel()
except RuntimeError:
# No running loop - nothing to clean up
pass
except Exception:
pass # Best effort cleanup
@pytest.fixture(autouse=True, scope="function")
def prevent_subprocess_orphans():
"""Track and clean up any subprocesses spawned during tests.
This prevents tests from leaving orphan processes that can cause
pytest-xdist workers to hang during shutdown.
"""
import subprocess
import weakref
# Track any Popen objects created during the test
original_popen = subprocess.Popen
spawned_processes = []
class TrackedPopen(original_popen):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
spawned_processes.append(weakref.ref(self))
subprocess.Popen = TrackedPopen
yield
# Restore original Popen
subprocess.Popen = original_popen
# Kill any processes that are still running
for proc_ref in spawned_processes:
proc = proc_ref()
if proc is not None:
try:
if proc.poll() is None: # Still running
proc.kill()
proc.wait(timeout=1)
except Exception:
pass # Best effort cleanup
@pytest.fixture(autouse=True)
def _stub_authentication_service_attestation(request, monkeypatch):
"""Prevent AuthenticationService from doing real CIRISVerify work in tests.
The production startup path launches a background task that performs FFI
calls into libtss2 and a live HTTPS registry lookup against
registry-us.ciris-services-1.ai. That's correct in production — failure
is fatal there — but in tests it (a) wastes seconds on every fixture that
starts the service, (b) issues a real lookup for an unreleased version
that the registry will 404, and (c) the orphan background task can outlive
individual tests and destabilize pytest-xdist workers.
This autouse fixture replaces `run_startup_attestation` and
`_attestation_refresh_loop` with deterministic no-ops that pre-populate
`_attestation_cache` with a synthetic AttestationResult, so any consumer
that calls `await_attestation_ready()` or `get_cached_attestation()`
still gets a valid object — just one constructed locally instead of
derived from live attestation.
Tests that need the real method bodies (e.g. tests that *test* the
refresh loop or startup-baseline capture themselves) opt out via
`@pytest.mark.real_attestation` at the test or module level.
"""
if request.node.get_closest_marker("real_attestation"):
return
import asyncio
from datetime import datetime, timezone
from ciris_engine.logic.services.infrastructure.authentication.service import (
AuthenticationService,
)
from ciris_engine.schemas.services.attestation import AttestationResult
def _synthetic_result() -> AttestationResult:
return AttestationResult(
loaded=True,
version="test-stub",
agent_version="test-stub",
hardware_type="SOFTWARE_ONLY",
key_status="ephemeral",
attestation_status="verified",
binary_ok=True,
env_ok=True,
file_integrity_ok=True,
audit_ok=True,
registry_ok=True,
python_integrity_ok=True,
module_integrity_ok=True,
max_level=4,
attestation_mode="full",
function_integrity="verified",
cached_at=datetime.now(timezone.utc),
cache_ttl_seconds=3600,
)
async def _stub_run_startup_attestation(self): # type: ignore[no-untyped-def]
result = _synthetic_result()
self._attestation_cache = result
self._last_known_attestation = result
self._baseline_attestation = result
async def _stub_attestation_refresh_loop(self): # type: ignore[no-untyped-def]
# Honour cancellation but do nothing — the refresh loop has no role
# in a test environment where the cache is set once at start().
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
raise
monkeypatch.setattr(
AuthenticationService, "run_startup_attestation", _stub_run_startup_attestation
)
monkeypatch.setattr(
AuthenticationService, "_attestation_refresh_loop", _stub_attestation_refresh_loop
)
@pytest.fixture(autouse=True)
def skip_without_discord_token(request):
"""Skip tests that require Discord token if not available."""
if request.node.get_closest_marker("requires_discord_token"):
if not os.environ.get("DISCORD_BOT_TOKEN"):
pytest.skip("Test requires DISCORD_BOT_TOKEN environment variable")
# SDK client fixture removed - ciris_sdk no longer exists
# Remove the event_loop fixture - let pytest-asyncio handle it
# The asyncio_mode = auto in pytest.ini will create event loops as needed
# Import centralized fixtures to make them available to all tests
pytest_plugins = [
"tests.fixtures.auth", # Centralized auth fixtures (SECURITY: uses dynamic passwords)
"tests.fixtures.telemetry_api",
"tests.fixtures.system_snapshot_fixtures",
"tests.fixtures.audit",
"tests.fixtures.database_maintenance",
]
@pytest.fixture
def api_required():
"""Mark test as requiring running API."""
import socket
# Check if API is accessible
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(1)
result = sock.connect_ex(("localhost", 8080))
sock.close()
if result != 0:
pytest.skip("API not running on localhost:8080")
except Exception:
pytest.skip("Cannot check API availability")
_EXIT_DEADLINE_SECONDS = int(os.environ.get("CIRIS_TEST_EXIT_DEADLINE_SECONDS", "90"))
def _arm_exit_deadline(exitstatus: int) -> None:
"""Force the process out if finalization is still running N seconds from now.
Armed at sessionfinish, after the terminal summary and after xdist has
shipped this worker's results, so nothing the run produced is discarded.
Daemon thread on purpose: it must never be the reason a process stays up.
A healthy run exits during normal finalization and this never fires, so
coverage writing and other atexit work are untouched.
Writes stacks to a FILE as well as stderr. When CI kills the job, the tail
of the step log is the first thing lost, and stderr from an xdist worker may
never be relayed at all — a file survives as an uploadable artifact and is
the only reliable way to see what the next hang was parked on.
"""
if _EXIT_DEADLINE_SECONDS <= 0:
return
def _bail() -> None:
time.sleep(_EXIT_DEADLINE_SECONDS)
role = _role()
path = os.environ.get("CIRIS_TEST_HANG_DUMP_DIR", ".")
try:
os.makedirs(path, exist_ok=True)
with open(os.path.join(path, f"hang-stacks-{role}.txt"), "w", encoding="utf-8") as fh:
fh.write(f"# finalization still running {_EXIT_DEADLINE_SECONDS}s after sessionfinish\n")
fh.write(f"# role={role} exitstatus={exitstatus} live_threads={threading.active_count()}\n")
for t in threading.enumerate():
fh.write(f"# {t.name} daemon={t.daemon} type={type(t).__name__}\n")
fh.flush()
faulthandler.dump_traceback(file=fh, all_threads=True)
except Exception: # noqa: BLE001 — diagnostics must never mask the exit
pass
_emit_stacks(
f"\n[conftest #956] {role}: interpreter still finalizing "
f"{_EXIT_DEADLINE_SECONDS}s after sessionfinish — forcing exit({exitstatus}).\n"
f"[conftest #956] {threading.active_count()} live threads; "
f"stacks also in hang-stacks-{role}.txt\n"
)
os._exit(exitstatus)
threading.Thread(target=_bail, name="ciris-exit-deadline", daemon=True).start()
def _role() -> str:
"""Which process this is, for the #956 guard's diagnostics.
Was referenced by the guard below but never defined — so on the very path
the guard exists for (lingering non-daemon threads at sessionfinish) it
raised NameError instead of dumping stacks and force-exiting, defeating its
own purpose. A test that leaves a default-executor worker alive surfaced it.
"""
return os.environ.get("PYTEST_XDIST_WORKER", "controller")
@pytest.hookimpl(trylast=True)
def pytest_sessionfinish(session, exitstatus): # noqa: D401
"""Guarantee the process exits once the run is genuinely over (#956).
THE PROBLEM: a full `pytest -n 16 tests/` executed the entire suite — ~27.8k
tests, 0 failures — and then never terminated. No summary, no exit code. The
xdist controller sat with 97 threads: 16 execnet receiver threads blocked in
`anon_pipe_read` on workers that had already exited, plus a thread parked in
`inet_csk_accept` holding a LISTENING socket with live ESTABLISHED clients.
A non-daemon thread blocked in accept() keeps the interpreter alive forever,
so teardown could not complete. Two runs hung this way, 50 and 90 minutes,
before being killed.
The suite PASSED both times. What was lost was the ability to say so: no
developer could get a full-suite verdict locally, and every pass/fail count
in this repo had to be read out of raw log markers instead of a summary line.
THE FIX: this hook runs AFTER the terminal summary is written and after
exitstatus is decided, so forcing the exit here discards nothing the run
produced. It names any non-daemon threads still alive first — a leaked
listener should be FIXED at its source, and a silent force-exit would hide
it exactly the way the hang did. Then it exits with the real status.
Deliberately controller-only: xdist workers have their own teardown protocol
and must be left alone.
"""
import threading
# RUNS IN WORKERS TOO. The first version skipped them ("not ours to end"),
# which protected the only process that was not stuck: measured live, all 16
# workers were ALIVE with 96 threads each, 94 of them parked in
# futex_do_wait, each still holding its socket to the controller. The
# controller's 16 blocked pipe-readers were the SYMPTOM — it was correctly
# waiting for workers that could never exit.
#
# (The earlier "workers are gone" reading came from counting them with
# `grep popen-gw`; execnet workers run as `python3 -u -c import sys;exec(...)`
# and never match that pattern. The count was zero because the probe was
# wrong, not because the workers had died.)
#
# trylast=True so xdist's own sessionfinish — which ships this worker's
# results back to the controller — has already run. Forcing the exit after
# that discards no test outcome.
# DEADLINE FIRST — armed unconditionally, before the non-daemon check below.
#
# That check is necessary but NOT sufficient, and shard 3 proved it on
# 2026-08-02: the run reached 99% PASSED, then hung 13 minutes until CI
# killed it. Measured at the hang, the workers held 98-131 live threads and
# EVERY leaked one was a `threading._DummyThread` — the placeholder Python
# fabricates when a thread it did not create calls in, i.e. an alien OS
# thread from a native extension. `_DummyThread.daemon` is True by
# definition, so `not t.daemon` filtered out precisely the threads that
# were present, `lingering` came back empty, and this hook returned and let
# the interpreter hang.
#
# Daemon does not mean harmless here. Python can kill a *Python* daemon
# thread when it next takes the GIL; it cannot do anything about an alien
# thread parked in native code, which can wedge finalization regardless of
# what the placeholder object claims.
#
# So: arm a deadline instead of trying to classify threads. It is a daemon
# thread, so it never blocks exit itself. A healthy run finishes
# finalization in well under a second and the process is gone long before
# it fires — which keeps pytest-cov's XML write and every other atexit hook
# on the normal path. Only a genuinely wedged finalization reaches it, and
# that one gets stacks plus the real exit status instead of a silent kill.
_arm_exit_deadline(exitstatus)
lingering = [
t
for t in threading.enumerate()
if t is not threading.current_thread() and t.is_alive() and not t.daemon
]
if not lingering:
return # nothing obviously leaked; the deadline above covers the rest
sys.stderr.write(
f"\n[conftest #956] {_role()}: {len(lingering)} non-daemon thread(s) still alive after "
f"sessionfinish; the interpreter would hang. Forcing exit({exitstatus}).\n"
f"[conftest #956] These are LEAKS — fix them at the source:\n"
)
for t in lingering[:20]:
sys.stderr.write(f"[conftest #956] - {t.name} (ident={t.ident})\n")
# Dump the STACK of every lingering thread, not just its name. A name like
# "Thread-7" is unactionable; the frame it is parked in (e.g. a CIRISVerify
# FFI call, socket.accept) names the source to fix. Cheap, and only runs on
# the leak path.
sys.stderr.write("[conftest #956] --- stacks of all threads (find the blocked frame) ---\n")
sys.stderr.flush()
try:
faulthandler.dump_traceback(all_threads=True)
except Exception: # noqa: BLE001
pass
try:
sys.stdout.flush()
except Exception: # noqa: BLE001
pass
os._exit(exitstatus)

Xet Storage Details

Size:
41.9 kB
·
Xet hash:
33387a59daabafda5ee2e22294223f4424d8d4a0d65e9eab7649fcb38d729074

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.