Buckets:
| """ | |
| Global test configuration for pytest. | |
| This file is automatically loaded by pytest and contains setup that applies to all tests. | |
| """ | |
| # CRITICAL: Set import protection BEFORE any other imports | |
| import faulthandler | |
| import os | |
| import signal | |
| import sys | |
| import tempfile | |
| import threading | |
| import time | |
| # #956 DIAGNOSTIC — make a hang self-report instead of dying silently. | |
| # | |
| # The intermittent pytest-xdist exit-hang was root-caused (non-daemon FFI | |
| # threads in setup/attestation.py that leaked past their join timeout), but a | |
| # leak that recurs from a NEW source must not cost another 30-minute silent | |
| # timeout to diagnose. faulthandler here dumps EVERY thread's stack — naming the | |
| # blocked frame's file:line — in two situations the sessionfinish guard below | |
| # cannot cover on its own: | |
| # * on a fatal signal (segfault etc.), and | |
| # * on SIGTERM, which is exactly how CI kills the hang: the shard wraps pytest | |
| # in `timeout --signal=SIGTERM ... 1800`, so registering here converts the | |
| # 30-min silent death into a full per-thread traceback for whichever process | |
| # received the signal (at minimum the controller). | |
| # Pure diagnostics: fires only on an actual signal, never during a healthy run. | |
| faulthandler.enable() | |
| if hasattr(faulthandler, "register") and hasattr(signal, "SIGTERM"): | |
| try: | |
| faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) | |
| except (ValueError, OSError): | |
| # Some environments disallow registering handlers (e.g. non-main thread | |
| # at import). Diagnostics are best-effort; never break collection. | |
| pass | |
| # WATCHDOG DUMP — the SIGTERM handler above proved insufficient: when CI's | |
| # job-level timeout-minutes fires, GitHub tears the process down fast enough | |
| # that a signal-driven dump does not reach the captured log. This is | |
| # signal-independent: a dedicated timer thread dumps EVERY thread's stack after | |
| # a fixed wall-clock interval, straight to fd 2, so a hung shard self-reports | |
| # INTO THE LIVE LOG well before it is killed. | |
| # | |
| # 1440s (24 min): healthy test shards finish in ~17-19 min and exit — the timer | |
| # dies with the process, no dump. A hung shard (killed at the 30-min job | |
| # timeout) gets its stacks dumped at 24 min, 6 min before death, while the step | |
| # log is still being tailed. Chosen to sit ABOVE the slowest healthy shard and | |
| # BELOW the kill. repeat=True in case the first dump races teardown. | |
| # | |
| # This is how we find the residual leak the attestation daemonization did NOT | |
| # fix (a non-thread mechanism — leaked child process holding the execnet fd, or | |
| # a C-extension thread) — it will name the exact blocked frame next hang. | |
| # | |
| # MEASURED 2026-08-02: `faulthandler.dump_traceback_later()` DOES NOT SURVIVE A | |
| # PYTEST RUN. Set the interval to 3s and run a 9s suite and you get zero dumps — | |
| # pytest's own faulthandler plugin calls `faulthandler.enable(file=<dup'd fd>)` | |
| # at configure time and the pending timer is lost. So this watchdog never fired, | |
| # which is why the 2026-08-02 shard-3 hang produced no stacks at all despite | |
| # being armed for exactly that. | |
| # | |
| # Replaced with a plain daemon timer thread calling `dump_traceback()` directly. | |
| # It is ours, nothing else touches it, and it is a daemon so it cannot itself | |
| # hold the process open. | |
| # A dup of the REAL stderr, taken at import before pytest installs capture. | |
| # `sys.stderr` is a capture buffer for most of a run, so anything a watchdog | |
| # writes there lands in a buffer that a hung or killed process never flushes — | |
| # which is the second reason the previous watchdog produced nothing. pytest's | |
| # own faulthandler plugin dups the fd for exactly this reason. | |
| try: | |
| _REAL_STDERR_FD = os.dup(2) | |
| except OSError: # pragma: no cover | |
| _REAL_STDERR_FD = 2 | |
| def _emit_stacks(header: str, tag: str = "hang") -> None: | |
| """Write a header + every thread's stack to a FILE, and try stderr too. | |
| THE FILE IS THE RELIABLE CHANNEL, and that is not a preference — pytest | |
| installs fd-level capture (``--capture=fd`` is the default) BEFORE conftest | |
| is imported, so even a dup of fd 2 taken here is a dup of pytest's capture | |
| pipe rather than the terminal. Anything written there lands in a buffer that | |
| a hung or SIGKILLed process never flushes. Measured: a 3s watchdog on a 9s | |
| suite produced no visible output through any stderr route. | |
| So write a file first and unconditionally. CI uploads it as an artifact, | |
| which also survives the step log being truncated at the tail — exactly what | |
| happened to the 2026-08-02 shard-3 hang, where nothing at all was captured. | |
| The stderr attempt is kept as a bonus for `-s` / local runs. | |
| """ | |
| path = os.environ.get("CIRIS_TEST_HANG_DUMP_DIR", ".") | |
| role = os.environ.get("PYTEST_XDIST_WORKER", "controller") | |
| try: | |
| os.makedirs(path, exist_ok=True) | |
| with open(os.path.join(path, f"{tag}-stacks-{role}.txt"), "a", encoding="utf-8") as fh: | |
| fh.write(header) | |
| fh.write(f"# live_threads={threading.active_count()}\n") | |
| for t in threading.enumerate(): | |
| fh.write(f"# {t.name} daemon={t.daemon} type={type(t).__name__}\n") | |
| fh.flush() | |
| faulthandler.dump_traceback(file=fh, all_threads=True) | |
| fh.flush() | |
| os.fsync(fh.fileno()) | |
| except Exception: # noqa: BLE001 — diagnostics must never break the run | |
| pass | |
| try: | |
| os.write(_REAL_STDERR_FD, header.encode("utf-8", "replace")) | |
| with os.fdopen(os.dup(_REAL_STDERR_FD), "w", closefd=True) as fh: | |
| faulthandler.dump_traceback(file=fh, all_threads=True) | |
| fh.flush() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| _HANG_WATCHDOG_SECONDS = int(os.environ.get("CIRIS_TEST_HANG_WATCHDOG_SECONDS", "1440")) | |
| if _HANG_WATCHDOG_SECONDS > 0: | |
| def _hang_watchdog() -> None: | |
| while True: | |
| time.sleep(_HANG_WATCHDOG_SECONDS) | |
| _emit_stacks( | |
| f"\n[conftest #956] WATCHDOG: still running after " | |
| f"{_HANG_WATCHDOG_SECONDS}s — every thread's stack follows.\n" | |
| ) | |
| try: | |
| threading.Thread(target=_hang_watchdog, name="ciris-hang-watchdog", daemon=True).start() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| # CONTROLLER DEADLINE — the bound the 2026-08-02 main-branch hang proved missing. | |
| # | |
| # Run 30754955268, shard 2: gw1/2/3 finished, hit the sessionfinish deadline | |
| # below, and self-exited — that mitigation WORKED. gw0 wedged inside a single | |
| # test (`test_database_maintenance_multi_occurrence.py::TestAdapterConfigHelpers:: | |
| # test_find_adapter_ids_from_configs_ignores_non_type_keys`, [TEST_START] with no | |
| # result) at native level: pytest-timeout's 60s thread-method timer never got the | |
| # GIL, which points at the Rust persist/sqlx teardown race documented in | |
| # tests/fixtures/database.py:12. The controller then sat in | |
| # `dsession.py loop_once -> queue.get()` for 9.5 minutes until the EXTERNAL | |
| # `timeout` SIGTERM'd it at 1500s (exit 124, no Python stacks from the bound | |
| # that killed it). | |
| # | |
| # Why none of the existing bounds could fire: | |
| # * `--session-timeout` is evaluated inside pytest-timeout's runtest_protocol | |
| # hookwrapper — WORKERS ONLY, and only between items. Once the last live | |
| # worker is wedged *inside* an item, nothing evaluates it ever again. It | |
| # bounds the loop only while workers are still completing items. | |
| # * `_arm_exit_deadline` (below) arms at pytest_sessionfinish — the controller | |
| # never got there. Its dump showed no ciris-exit-deadline thread. | |
| # * pytest-timeout's per-test timer needs the GIL; a native block starves it. | |
| # | |
| # This deadline is wall-clock, armed at import in the CONTROLLER ONLY, and | |
| # OPT-IN VIA ENV (default off): a healthy local full-suite run can legitimately | |
| # exceed any number we would pick here, so the number belongs to the CI job that | |
| # knows its own budget. CI sets it just below the external SIGTERM so the | |
| # process self-reports with stacks and a real exit code instead of dying | |
| # silently 2 minutes later. | |
| _CONTROLLER_DEADLINE_SECONDS = int(os.environ.get("CIRIS_TEST_CONTROLLER_DEADLINE_SECONDS", "0")) | |
| if _CONTROLLER_DEADLINE_SECONDS > 0 and "PYTEST_XDIST_WORKER" not in os.environ: | |
| def _controller_deadline() -> None: | |
| time.sleep(_CONTROLLER_DEADLINE_SECONDS) | |
| _emit_stacks( | |
| f"\n[conftest #956] CONTROLLER DEADLINE: session still running after " | |
| f"{_CONTROLLER_DEADLINE_SECONDS}s — a worker is likely wedged in native code. " | |
| f"Dumping stacks and exiting 1 so the bound that killed the run is the one " | |
| f"that reported it.\n" | |
| ) | |
| os._exit(1) | |
| try: | |
| threading.Thread(target=_controller_deadline, name="ciris-controller-deadline", daemon=True).start() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| # BELT for the residual #956 leak: the CIRISVerify ThreadPoolExecutor. | |
| # | |
| # concurrent.futures registers `_python_exit` via atexit to JOIN every executor | |
| # worker thread at interpreter shutdown (executor workers are non-daemon since | |
| # 3.9). ciris_verify/ffi_bindings/client.py holds a ThreadPoolExecutor and | |
| # submits CIRISVerify FFI calls to it. In CI there is no attestation hardware, | |
| # so an FFI task can block indefinitely — and `shutdown(wait=False)` in the | |
| # client's __del__ cannot cancel a task already running in a worker. That leaves | |
| # a non-daemon worker stuck in the FFI, and `_python_exit` blocks joining it | |
| # FOREVER at shutdown. It is invisible to the threading.Thread AST lint (the | |
| # threads are stdlib-created) and it runs in the atexit phase, AFTER the | |
| # sessionfinish guard, which is why the guard did not catch it. | |
| # | |
| # The test process does not need in-flight executor results at exit, so drop the | |
| # join. The sessionfinish guard still force-exits on any lingering non-daemon | |
| # thread; this removes the one blocking phase that runs past it. Belt only — the | |
| # watchdog above still fires if a DIFFERENT residual mechanism remains, so this | |
| # cannot mask a further leak. | |
| try: | |
| import atexit as _atexit | |
| import concurrent.futures.thread as _cf_thread | |
| _atexit.unregister(_cf_thread._python_exit) | |
| except Exception: # noqa: BLE001 — best-effort; never break collection | |
| pass | |
| os.environ["CIRIS_IMPORT_MODE"] = "true" | |
| os.environ["CIRIS_MOCK_LLM"] = "true" | |
| os.environ["CIRIS_TESTING_MODE"] = "true" # Enable fallback admin credentials for tests | |
| # Unit tests never boot the real node fold (ciris-server serve on 4243): since | |
| # 0.5.113 the fold is fail-fast (node-fails => agent-fails) and requires a live | |
| # embedded edge (init_edge_runtime) + SQLite engine, which bare unit tests do | |
| # not stand up. Staged QA / integration boots override this explicitly. | |
| os.environ.setdefault("CIRIS_NODE_FOLD", "false") | |
| # Pin language to English so handler-content assertions stay stable regardless | |
| # of the developer's local CIRIS_PREFERRED_LANGUAGE in .env. Tests that | |
| # intentionally exercise localization can override via monkeypatch. | |
| os.environ["CIRIS_PREFERRED_LANGUAGE"] = "en" | |
| # CRITICAL: Override log directory for tests to prevent container interference | |
| # Tests should NEVER write to the main logs directory that containers use | |
| os.environ["CIRIS_LOG_DIR"] = "test_logs" | |
| os.environ["CIRIS_DATA_DIR"] = "test_data" | |
| # PERFORMANCE: Use tmpfs for all temp files to reduce disk I/O during tests | |
| # /dev/shm is a RAM-based tmpfs available on most Linux systems | |
| # | |
| # Per-worker isolation (added 2026-04-30 for the -n 28 parallel-execution | |
| # regression in db-maintenance-multi-occurrence + dsar-multi-source): | |
| # every worker gets its OWN TMPDIR subdir under _TMPFS_BASE. Without this, | |
| # all workers shared `/dev/shm/ciris_tests/` and pytest-xdist's session- | |
| # level cleanup of `pytest-of-emoore/pytest-0/popen-gw{N}/` could delete | |
| # one worker's tmpdir while another worker was still using it (FileNotFoundError | |
| # on iterdir during test setup, observed in -n 28 runs). | |
| _TMPFS_BASE = "/dev/shm/ciris_tests" | |
| if os.path.isdir("/dev/shm"): | |
| import shutil | |
| is_xdist_worker = os.environ.get("PYTEST_XDIST_WORKER") is not None | |
| # INVARIANT: no process deletes a directory it does not exclusively own. | |
| # | |
| # The previous version gave the non-worker process `_TMPFS_BASE` ITSELF and | |
| # rmtree'd it. Because every worker's dir is a CHILD of that base, any | |
| # process importing this conftest without PYTEST_XDIST_WORKER set — the | |
| # xdist controller included — destroyed every live worker's TMPDIR | |
| # mid-session. Workers then failed in `tempfile.mkdtemp()` itself, so | |
| # switching a test away from `tmp_path` did NOT help; the failure counts | |
| # varied run to run (302-412 errors) and made every `-n` result unreadable. | |
| # See #947. | |
| if is_xdist_worker: | |
| worker_id = os.environ["PYTEST_XDIST_WORKER"] # e.g. "gw23" | |
| _TMPFS_DIR = os.path.join(_TMPFS_BASE, worker_id) | |
| else: | |
| # The controller/serial process gets its OWN subdir, a sibling of the | |
| # workers' rather than their parent. | |
| _TMPFS_DIR = os.path.join(_TMPFS_BASE, "main") | |
| # Stale-sweep, not an eager wipe: the stated goal is keeping /dev/shm from | |
| # filling up over time, which an age check satisfies without touching | |
| # anything a concurrent run is using. Only the base's direct children are | |
| # considered, and only when nothing has written to them for an hour. | |
| if not is_xdist_worker and os.path.isdir(_TMPFS_BASE): | |
| _stale_cutoff = time.time() - 3600 | |
| for _entry in os.listdir(_TMPFS_BASE): | |
| _path = os.path.join(_TMPFS_BASE, _entry) | |
| try: | |
| if os.path.getmtime(_path) < _stale_cutoff: | |
| shutil.rmtree(_path, ignore_errors=True) | |
| except OSError: | |
| pass # racing another run's cleanup is fine; it owns that dir | |
| os.makedirs(_TMPFS_DIR, exist_ok=True) | |
| os.environ["TMPDIR"] = _TMPFS_DIR | |
| tempfile.tempdir = _TMPFS_DIR | |
| # PERFORMANCE: Disable network-dependent services in tests | |
| # This prevents NTP checks, billing API calls, etc. | |
| os.environ["CIRIS_DISABLE_NETWORK"] = "true" | |
| # NOTE: Network calls are disabled via CIRIS_IMPORT_MODE and CIRIS_MOCK_LLM | |
| # Code should check these flags and skip network operations entirely | |
| from pathlib import Path # noqa: E402 | |
| import pytest # noqa: E402 | |
| # Load environment variables from .env file for all tests | |
| try: | |
| from dotenv import load_dotenv | |
| # Find the .env file in the project root | |
| project_root = Path(__file__).parent.parent | |
| env_file = project_root / ".env" | |
| if env_file.exists(): | |
| load_dotenv(env_file) | |
| # Also load test-specific config if available | |
| test_env_file = Path(__file__).parent / "test_config.env" | |
| if test_env_file.exists(): | |
| load_dotenv(test_env_file, override=True) | |
| # Also set as environment variables for non-dotenv aware code | |
| import subprocess | |
| result = subprocess.run(["bash", "-c", f"source {test_env_file} && env"], capture_output=True, text=True) | |
| if result.returncode == 0: | |
| for line in result.stdout.splitlines(): | |
| if "=" in line and line.startswith("CIRIS_"): | |
| key, value = line.split("=", 1) | |
| os.environ[key] = value | |
| except ImportError: | |
| # If python-dotenv is not installed, silently continue | |
| pass | |
| import gc # noqa: E402 | |
| # Import database fixtures and API fixtures - must be imported after os.environ setup | |
| from tests.fixtures.api import random_api_port # noqa: E402 | |
| from tests.fixtures.database import ( # noqa: E402 | |
| _release_persist_engine, | |
| clean_db, | |
| test_db, | |
| ) | |
| from tests.fixtures.mocks import MockRuntime # noqa: E402 | |
| from tests.fixtures.persist_engine import persist_engine # noqa: E402, F401 | |
| from tests.fixtures.runtime_control import ( # noqa: E402 | |
| mock_api_runtime_control_service, | |
| mock_step_result_gather_context, | |
| mock_step_result_perform_dmas, | |
| single_step_control_response, | |
| ) | |
| def mock_runtime(mock_step_result_perform_dmas): | |
| """Create a centralized MockRuntime with step result data.""" | |
| runtime = MockRuntime() | |
| # Connect the centralized step result fixture | |
| runtime.pipeline_controller.get_latest_step_result.return_value = mock_step_result_perform_dmas | |
| return runtime | |
| # Make fixtures available to all tests by explicitly referencing them | |
| __all__ = [ | |
| "test_db", | |
| "clean_db", | |
| "random_api_port", | |
| "single_step_control_response", | |
| "mock_step_result_perform_dmas", | |
| "mock_step_result_gather_context", | |
| "mock_api_runtime_control_service", | |
| "mock_runtime", | |
| ] | |
| def manage_import_protection(): | |
| """Manage import protection for the entire test session.""" | |
| # Import protection is already set at module level | |
| # But we ensure it stays set during test collection | |
| os.environ["CIRIS_IMPORT_MODE"] = "true" | |
| os.environ["CIRIS_MOCK_LLM"] = "true" | |
| yield | |
| # After all tests are done, we can clear the protection | |
| # (though it doesn't really matter since process is ending) | |
| os.environ.pop("CIRIS_IMPORT_MODE", None) | |
| def _isolate_process_global_env(monkeypatch): | |
| """Restore global, process-wide env vars to known values for every test. | |
| Some tests intentionally flip `CIRIS_PREFERRED_LANGUAGE` or | |
| `CIRIS_LLM_REPLICAS` via raw `os.environ[...] = ...` to exercise | |
| behavior that depends on those globals. Those env vars work fine for | |
| personal-agent deployments (one process, one operator) but they are | |
| the wrong shape for a test harness: under `pytest -n`, xdist workers | |
| run tests serially per worker, so a leaked mutation from test A can | |
| change the behavior of unrelated test B and produce flaky CI | |
| failures (e.g. a handler follow-up rendered in Amharic, or a second | |
| LLM replica being initialized when the test expects one). | |
| Using monkeypatch here captures the current values before each test | |
| and restores them at teardown — even if a test mutates them with raw | |
| `os.environ[...]` and forgets to restore, or crashes mid-test. Tests | |
| that want a different value should still prefer `monkeypatch.setenv` | |
| inside the test so the change is scoped and obvious. | |
| """ | |
| monkeypatch.setenv("CIRIS_PREFERRED_LANGUAGE", "en") | |
| monkeypatch.delenv("CIRIS_LLM_REPLICAS", raising=False) | |
| def pytest_runtest_setup(item): # noqa: ARG001 | |
| """Reset the process-wide persist Engine before every test. | |
| ciris-persist's Engine is a process-singleton: a second construction | |
| with a different DSN raises EngineConfigMismatch, and the test suite | |
| has ~20+ fixtures that each bootstrap a per-test engine. | |
| This is a hook, not an autouse fixture, deliberately: a `tryfirst` | |
| `pytest_runtest_setup` runs strictly BEFORE all fixture setup — | |
| including class-level autouse fixtures that wire their own engine — | |
| so no fixture-ordering race can leave a prior test's engine pinned. | |
| `_release_persist_engine` calls ciris-persist's `reset_engine()` | |
| (CIRISPersist#88), which un-pins the singleton handle-free — so even | |
| an orphan engine (global nulled without close) is recovered. | |
| """ | |
| _release_persist_engine() | |
| def pytest_runtest_teardown(item): # noqa: ARG001 | |
| """Release the persist Engine after every test (mirror of setup).""" | |
| _release_persist_engine() | |
| def cleanup_after_test(request): | |
| """ | |
| Ensure proper cleanup after each test. | |
| This helps prevent interference between tests, especially when Discord is involved. | |
| """ | |
| yield | |
| # Force garbage collection to clean up any lingering objects | |
| gc.collect() | |
| # Only add socket cleanup delay for tests that actually use network/Discord | |
| # This saves ~0.1s per test for the majority of tests that don't need it | |
| test_path = str(request.fspath) if hasattr(request, "fspath") else "" | |
| needs_socket_cleanup = ( | |
| "discord" in test_path.lower() | |
| or request.node.get_closest_marker("requires_discord_token") | |
| or request.node.get_closest_marker("needs_socket_cleanup") | |
| ) | |
| if needs_socket_cleanup: | |
| time.sleep(0.1) | |
| def strip_leaked_incident_handlers(): | |
| """Remove any IncidentCaptureHandler left on a process-global logger after a | |
| test, so it cannot leak onto the next test sharing the xdist worker process. | |
| Root cause of the recurring 'incident_handler_injection' / logging-pollution | |
| flake (git: e8a16d764, 3f254498f): ``add_incident_capture_handler()`` does | |
| ``target_logger.addHandler(handler)`` on a process-global logger, and not | |
| every test removes it. Under pytest-xdist the leaked handler then shows up on | |
| the next test's logger — e.g. ``assert len(specific_logger.handlers) == 0`` | |
| sees 2. Stripping the leak at teardown fixes it at the source instead of | |
| per-test ``xdist_group`` whack-a-mole. Surgical: only the IncidentCaptureHandler | |
| subclass is removed, so ``caplog`` and other handlers are left untouched. | |
| """ | |
| yield | |
| import logging as _logging | |
| try: | |
| from ciris_engine.logic.utils.incident_capture_handler import IncidentCaptureHandler | |
| except Exception: # pragma: no cover - module always importable in-tree | |
| return | |
| loggers = [_logging.getLogger()] | |
| loggers += [_logging.getLogger(name) for name in list(_logging.Logger.manager.loggerDict)] | |
| for lg in loggers: | |
| handlers = getattr(lg, "handlers", None) | |
| if not handlers: | |
| continue | |
| kept = [h for h in handlers if not isinstance(h, IncidentCaptureHandler)] | |
| if len(kept) != len(handlers): | |
| lg.handlers = kept | |
| def clear_asyncio_module_state(): | |
| """ | |
| Clear module-level asyncio state to prevent cross-test contamination. | |
| This fixes the "asyncio.Event/Queue object is bound to a different event loop" error | |
| that occurs when pytest-xdist runs tests in parallel with different event loops. | |
| Affected modules: | |
| 1. step_decorators: | |
| - _paused_thoughts: Dict[str, asyncio.Event] - Events bound to specific event loops | |
| - _single_step_mode: bool - Global flag for single-step debugging | |
| 2. step_streaming: | |
| - reasoning_event_stream._subscribers: set[asyncio.Queue] - Queues bound to event loops | |
| - reasoning_event_stream._sequence_number: int - Counter that should reset | |
| - reasoning_event_stream._recent_events: list - Buffer that should clear | |
| Without this cleanup, asyncio primitives created in one test's event loop persist | |
| and cause RuntimeError when accessed from a different test's event loop. | |
| """ | |
| def _clear_state(): | |
| # Clear step_decorators state | |
| try: | |
| from ciris_engine.logic.processors.core import step_decorators | |
| step_decorators._paused_thoughts.clear() | |
| step_decorators._single_step_mode = False | |
| except ImportError: | |
| pass # Module not yet available during collection | |
| # Clear step_streaming singleton state | |
| try: | |
| from ciris_engine.logic.infrastructure.step_streaming import reasoning_event_stream | |
| reasoning_event_stream._subscribers.clear() | |
| reasoning_event_stream._sequence_number = 0 | |
| reasoning_event_stream._recent_events.clear() | |
| except ImportError: | |
| pass # Module not yet available during collection | |
| # Clear before test | |
| _clear_state() | |
| yield | |
| # Clear after test to ensure clean state for next test | |
| _clear_state() | |
| def isolate_test_env_vars(): | |
| """ | |
| Isolate environment variables that can cause test pollution. | |
| This fixture saves any existing env vars before the test and restores | |
| them after, ensuring parallel test runs don't interfere with each other. | |
| """ | |
| # Env vars that need isolation | |
| env_vars = [ | |
| # Accord metrics (current names) | |
| "CIRIS_ACCORD_METRICS_CONSENT", | |
| "CIRIS_ACCORD_METRICS_CONSENT_TIMESTAMP", | |
| "CIRIS_ACCORD_METRICS_TRACE_LEVEL", | |
| "CIRIS_ACCORD_METRICS_FLUSH_INTERVAL", | |
| "CIRIS_ACCORD_METRICS_ENDPOINT", | |
| # Legacy covenant metrics (for backward compat) | |
| "CIRIS_COVENANT_METRICS_CONSENT", | |
| "CIRIS_COVENANT_METRICS_CONSENT_TIMESTAMP", | |
| "CIRIS_COVENANT_METRICS_TRACE_LEVEL", | |
| "CIRIS_COVENANT_METRICS_FLUSH_INTERVAL", | |
| "CIRIS_COVENANT_METRICS_ENDPOINT", | |
| # LLM provider detection (affects which provider is selected) | |
| "CIRIS_LLM_PROVIDER", | |
| "LLM_PROVIDER", | |
| "OPENAI_API_KEY", | |
| "OPENAI_API_BASE", | |
| "ANTHROPIC_API_KEY", | |
| "GOOGLE_API_KEY", | |
| "GEMINI_API_KEY", | |
| # Dual LLM service | |
| "CIRIS_OPENAI_API_KEY_2", | |
| "CIRIS_OPENAI_API_BASE_2", | |
| "CIRIS_OPENAI_MODEL_NAME_2", | |
| ] | |
| # Save existing values | |
| saved = {var: os.environ.get(var) for var in env_vars} | |
| # Clear them before test | |
| for var in env_vars: | |
| os.environ.pop(var, None) | |
| yield | |
| # Restore original values after test | |
| for var, value in saved.items(): | |
| if value is not None: | |
| os.environ[var] = value | |
| else: | |
| os.environ.pop(var, None) | |
| # Import WA test harness fixtures (when available) | |
| try: | |
| from ciris_engine.logic.services.test_wa_auth_harness import wa_test_env, wa_test_harness, wa_test_keys | |
| except ImportError: | |
| # WA test harness not available yet | |
| wa_test_harness = None | |
| wa_test_env = None | |
| wa_test_keys = None | |
| # Skip Discord tests if no token is set | |
| def pytest_configure(config): | |
| config.addinivalue_line("markers", "requires_discord_token: mark test as requiring Discord token") | |
| config.addinivalue_line("markers", "needs_socket_cleanup: mark test as needing socket cleanup delay") | |
| config.addinivalue_line( | |
| "markers", | |
| "real_attestation: opt out of the autouse stub that no-ops " | |
| "AuthenticationService.run_startup_attestation / _attestation_refresh_loop", | |
| ) | |
| # ============================================================================= | |
| # Test Start/End Markers for Debugging Hanging Tests | |
| # ============================================================================= | |
| # These hooks print markers that can be grep'd to find tests that started but | |
| # never finished (hanging tests). Look for [TEST_START] without matching [TEST_END]. | |
| def pytest_runtest_logstart(nodeid, location): | |
| """Print marker when test starts - helps identify hanging tests in CI logs.""" | |
| import sys | |
| # Flush to ensure marker appears immediately in CI logs | |
| print(f"\n[TEST_START] {nodeid}", file=sys.stderr, flush=True) | |
| def pytest_runtest_logfinish(nodeid, location): | |
| """Print marker when test finishes - pair with TEST_START to find hangs.""" | |
| import sys | |
| print(f"[TEST_END] {nodeid}", file=sys.stderr, flush=True) | |
| # ============================================================================ | |
| # CASCADING FAILURE PROTECTION | |
| # ============================================================================ | |
| # These hooks and fixtures prevent a single test crash from causing | |
| # cascading failures (e.g., 18-minute hangs due to orphan processes). | |
| def pytest_runtest_teardown(item, nextitem): | |
| """Clean up any orphan async tasks after each test. | |
| This prevents tests from leaving behind background tasks that can | |
| cause worker crashes or hangs in pytest-xdist. | |
| """ | |
| import asyncio | |
| try: | |
| # Only cancel tasks if there's a running event loop | |
| # get_event_loop() without a running loop is deprecated in Python 3.10+ | |
| # and there are no tasks to cancel anyway if there's no loop | |
| try: | |
| loop = asyncio.get_running_loop() | |
| # Cancel any pending tasks from this test | |
| pending = asyncio.all_tasks(loop) | |
| for task in pending: | |
| if not task.done(): | |
| task.cancel() | |
| except RuntimeError: | |
| # No running loop - nothing to clean up | |
| pass | |
| except Exception: | |
| pass # Best effort cleanup | |
| def prevent_subprocess_orphans(): | |
| """Track and clean up any subprocesses spawned during tests. | |
| This prevents tests from leaving orphan processes that can cause | |
| pytest-xdist workers to hang during shutdown. | |
| """ | |
| import subprocess | |
| import weakref | |
| # Track any Popen objects created during the test | |
| original_popen = subprocess.Popen | |
| spawned_processes = [] | |
| class TrackedPopen(original_popen): | |
| def __init__(self, *args, **kwargs): | |
| super().__init__(*args, **kwargs) | |
| spawned_processes.append(weakref.ref(self)) | |
| subprocess.Popen = TrackedPopen | |
| yield | |
| # Restore original Popen | |
| subprocess.Popen = original_popen | |
| # Kill any processes that are still running | |
| for proc_ref in spawned_processes: | |
| proc = proc_ref() | |
| if proc is not None: | |
| try: | |
| if proc.poll() is None: # Still running | |
| proc.kill() | |
| proc.wait(timeout=1) | |
| except Exception: | |
| pass # Best effort cleanup | |
| def _stub_authentication_service_attestation(request, monkeypatch): | |
| """Prevent AuthenticationService from doing real CIRISVerify work in tests. | |
| The production startup path launches a background task that performs FFI | |
| calls into libtss2 and a live HTTPS registry lookup against | |
| registry-us.ciris-services-1.ai. That's correct in production — failure | |
| is fatal there — but in tests it (a) wastes seconds on every fixture that | |
| starts the service, (b) issues a real lookup for an unreleased version | |
| that the registry will 404, and (c) the orphan background task can outlive | |
| individual tests and destabilize pytest-xdist workers. | |
| This autouse fixture replaces `run_startup_attestation` and | |
| `_attestation_refresh_loop` with deterministic no-ops that pre-populate | |
| `_attestation_cache` with a synthetic AttestationResult, so any consumer | |
| that calls `await_attestation_ready()` or `get_cached_attestation()` | |
| still gets a valid object — just one constructed locally instead of | |
| derived from live attestation. | |
| Tests that need the real method bodies (e.g. tests that *test* the | |
| refresh loop or startup-baseline capture themselves) opt out via | |
| `@pytest.mark.real_attestation` at the test or module level. | |
| """ | |
| if request.node.get_closest_marker("real_attestation"): | |
| return | |
| import asyncio | |
| from datetime import datetime, timezone | |
| from ciris_engine.logic.services.infrastructure.authentication.service import ( | |
| AuthenticationService, | |
| ) | |
| from ciris_engine.schemas.services.attestation import AttestationResult | |
| def _synthetic_result() -> AttestationResult: | |
| return AttestationResult( | |
| loaded=True, | |
| version="test-stub", | |
| agent_version="test-stub", | |
| hardware_type="SOFTWARE_ONLY", | |
| key_status="ephemeral", | |
| attestation_status="verified", | |
| binary_ok=True, | |
| env_ok=True, | |
| file_integrity_ok=True, | |
| audit_ok=True, | |
| registry_ok=True, | |
| python_integrity_ok=True, | |
| module_integrity_ok=True, | |
| max_level=4, | |
| attestation_mode="full", | |
| function_integrity="verified", | |
| cached_at=datetime.now(timezone.utc), | |
| cache_ttl_seconds=3600, | |
| ) | |
| async def _stub_run_startup_attestation(self): # type: ignore[no-untyped-def] | |
| result = _synthetic_result() | |
| self._attestation_cache = result | |
| self._last_known_attestation = result | |
| self._baseline_attestation = result | |
| async def _stub_attestation_refresh_loop(self): # type: ignore[no-untyped-def] | |
| # Honour cancellation but do nothing — the refresh loop has no role | |
| # in a test environment where the cache is set once at start(). | |
| try: | |
| await asyncio.Event().wait() | |
| except asyncio.CancelledError: | |
| raise | |
| monkeypatch.setattr( | |
| AuthenticationService, "run_startup_attestation", _stub_run_startup_attestation | |
| ) | |
| monkeypatch.setattr( | |
| AuthenticationService, "_attestation_refresh_loop", _stub_attestation_refresh_loop | |
| ) | |
| def skip_without_discord_token(request): | |
| """Skip tests that require Discord token if not available.""" | |
| if request.node.get_closest_marker("requires_discord_token"): | |
| if not os.environ.get("DISCORD_BOT_TOKEN"): | |
| pytest.skip("Test requires DISCORD_BOT_TOKEN environment variable") | |
| # SDK client fixture removed - ciris_sdk no longer exists | |
| # Remove the event_loop fixture - let pytest-asyncio handle it | |
| # The asyncio_mode = auto in pytest.ini will create event loops as needed | |
| # Import centralized fixtures to make them available to all tests | |
| pytest_plugins = [ | |
| "tests.fixtures.auth", # Centralized auth fixtures (SECURITY: uses dynamic passwords) | |
| "tests.fixtures.telemetry_api", | |
| "tests.fixtures.system_snapshot_fixtures", | |
| "tests.fixtures.audit", | |
| "tests.fixtures.database_maintenance", | |
| ] | |
| def api_required(): | |
| """Mark test as requiring running API.""" | |
| import socket | |
| # Check if API is accessible | |
| try: | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| sock.settimeout(1) | |
| result = sock.connect_ex(("localhost", 8080)) | |
| sock.close() | |
| if result != 0: | |
| pytest.skip("API not running on localhost:8080") | |
| except Exception: | |
| pytest.skip("Cannot check API availability") | |
| _EXIT_DEADLINE_SECONDS = int(os.environ.get("CIRIS_TEST_EXIT_DEADLINE_SECONDS", "90")) | |
| def _arm_exit_deadline(exitstatus: int) -> None: | |
| """Force the process out if finalization is still running N seconds from now. | |
| Armed at sessionfinish, after the terminal summary and after xdist has | |
| shipped this worker's results, so nothing the run produced is discarded. | |
| Daemon thread on purpose: it must never be the reason a process stays up. | |
| A healthy run exits during normal finalization and this never fires, so | |
| coverage writing and other atexit work are untouched. | |
| Writes stacks to a FILE as well as stderr. When CI kills the job, the tail | |
| of the step log is the first thing lost, and stderr from an xdist worker may | |
| never be relayed at all — a file survives as an uploadable artifact and is | |
| the only reliable way to see what the next hang was parked on. | |
| """ | |
| if _EXIT_DEADLINE_SECONDS <= 0: | |
| return | |
| def _bail() -> None: | |
| time.sleep(_EXIT_DEADLINE_SECONDS) | |
| role = _role() | |
| path = os.environ.get("CIRIS_TEST_HANG_DUMP_DIR", ".") | |
| try: | |
| os.makedirs(path, exist_ok=True) | |
| with open(os.path.join(path, f"hang-stacks-{role}.txt"), "w", encoding="utf-8") as fh: | |
| fh.write(f"# finalization still running {_EXIT_DEADLINE_SECONDS}s after sessionfinish\n") | |
| fh.write(f"# role={role} exitstatus={exitstatus} live_threads={threading.active_count()}\n") | |
| for t in threading.enumerate(): | |
| fh.write(f"# {t.name} daemon={t.daemon} type={type(t).__name__}\n") | |
| fh.flush() | |
| faulthandler.dump_traceback(file=fh, all_threads=True) | |
| except Exception: # noqa: BLE001 — diagnostics must never mask the exit | |
| pass | |
| _emit_stacks( | |
| f"\n[conftest #956] {role}: interpreter still finalizing " | |
| f"{_EXIT_DEADLINE_SECONDS}s after sessionfinish — forcing exit({exitstatus}).\n" | |
| f"[conftest #956] {threading.active_count()} live threads; " | |
| f"stacks also in hang-stacks-{role}.txt\n" | |
| ) | |
| os._exit(exitstatus) | |
| threading.Thread(target=_bail, name="ciris-exit-deadline", daemon=True).start() | |
| def _role() -> str: | |
| """Which process this is, for the #956 guard's diagnostics. | |
| Was referenced by the guard below but never defined — so on the very path | |
| the guard exists for (lingering non-daemon threads at sessionfinish) it | |
| raised NameError instead of dumping stacks and force-exiting, defeating its | |
| own purpose. A test that leaves a default-executor worker alive surfaced it. | |
| """ | |
| return os.environ.get("PYTEST_XDIST_WORKER", "controller") | |
| def pytest_sessionfinish(session, exitstatus): # noqa: D401 | |
| """Guarantee the process exits once the run is genuinely over (#956). | |
| THE PROBLEM: a full `pytest -n 16 tests/` executed the entire suite — ~27.8k | |
| tests, 0 failures — and then never terminated. No summary, no exit code. The | |
| xdist controller sat with 97 threads: 16 execnet receiver threads blocked in | |
| `anon_pipe_read` on workers that had already exited, plus a thread parked in | |
| `inet_csk_accept` holding a LISTENING socket with live ESTABLISHED clients. | |
| A non-daemon thread blocked in accept() keeps the interpreter alive forever, | |
| so teardown could not complete. Two runs hung this way, 50 and 90 minutes, | |
| before being killed. | |
| The suite PASSED both times. What was lost was the ability to say so: no | |
| developer could get a full-suite verdict locally, and every pass/fail count | |
| in this repo had to be read out of raw log markers instead of a summary line. | |
| THE FIX: this hook runs AFTER the terminal summary is written and after | |
| exitstatus is decided, so forcing the exit here discards nothing the run | |
| produced. It names any non-daemon threads still alive first — a leaked | |
| listener should be FIXED at its source, and a silent force-exit would hide | |
| it exactly the way the hang did. Then it exits with the real status. | |
| Deliberately controller-only: xdist workers have their own teardown protocol | |
| and must be left alone. | |
| """ | |
| import threading | |
| # RUNS IN WORKERS TOO. The first version skipped them ("not ours to end"), | |
| # which protected the only process that was not stuck: measured live, all 16 | |
| # workers were ALIVE with 96 threads each, 94 of them parked in | |
| # futex_do_wait, each still holding its socket to the controller. The | |
| # controller's 16 blocked pipe-readers were the SYMPTOM — it was correctly | |
| # waiting for workers that could never exit. | |
| # | |
| # (The earlier "workers are gone" reading came from counting them with | |
| # `grep popen-gw`; execnet workers run as `python3 -u -c import sys;exec(...)` | |
| # and never match that pattern. The count was zero because the probe was | |
| # wrong, not because the workers had died.) | |
| # | |
| # trylast=True so xdist's own sessionfinish — which ships this worker's | |
| # results back to the controller — has already run. Forcing the exit after | |
| # that discards no test outcome. | |
| # DEADLINE FIRST — armed unconditionally, before the non-daemon check below. | |
| # | |
| # That check is necessary but NOT sufficient, and shard 3 proved it on | |
| # 2026-08-02: the run reached 99% PASSED, then hung 13 minutes until CI | |
| # killed it. Measured at the hang, the workers held 98-131 live threads and | |
| # EVERY leaked one was a `threading._DummyThread` — the placeholder Python | |
| # fabricates when a thread it did not create calls in, i.e. an alien OS | |
| # thread from a native extension. `_DummyThread.daemon` is True by | |
| # definition, so `not t.daemon` filtered out precisely the threads that | |
| # were present, `lingering` came back empty, and this hook returned and let | |
| # the interpreter hang. | |
| # | |
| # Daemon does not mean harmless here. Python can kill a *Python* daemon | |
| # thread when it next takes the GIL; it cannot do anything about an alien | |
| # thread parked in native code, which can wedge finalization regardless of | |
| # what the placeholder object claims. | |
| # | |
| # So: arm a deadline instead of trying to classify threads. It is a daemon | |
| # thread, so it never blocks exit itself. A healthy run finishes | |
| # finalization in well under a second and the process is gone long before | |
| # it fires — which keeps pytest-cov's XML write and every other atexit hook | |
| # on the normal path. Only a genuinely wedged finalization reaches it, and | |
| # that one gets stacks plus the real exit status instead of a silent kill. | |
| _arm_exit_deadline(exitstatus) | |
| lingering = [ | |
| t | |
| for t in threading.enumerate() | |
| if t is not threading.current_thread() and t.is_alive() and not t.daemon | |
| ] | |
| if not lingering: | |
| return # nothing obviously leaked; the deadline above covers the rest | |
| sys.stderr.write( | |
| f"\n[conftest #956] {_role()}: {len(lingering)} non-daemon thread(s) still alive after " | |
| f"sessionfinish; the interpreter would hang. Forcing exit({exitstatus}).\n" | |
| f"[conftest #956] These are LEAKS — fix them at the source:\n" | |
| ) | |
| for t in lingering[:20]: | |
| sys.stderr.write(f"[conftest #956] - {t.name} (ident={t.ident})\n") | |
| # Dump the STACK of every lingering thread, not just its name. A name like | |
| # "Thread-7" is unactionable; the frame it is parked in (e.g. a CIRISVerify | |
| # FFI call, socket.accept) names the source to fix. Cheap, and only runs on | |
| # the leak path. | |
| sys.stderr.write("[conftest #956] --- stacks of all threads (find the blocked frame) ---\n") | |
| sys.stderr.flush() | |
| try: | |
| faulthandler.dump_traceback(all_threads=True) | |
| except Exception: # noqa: BLE001 | |
| pass | |
| try: | |
| sys.stdout.flush() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| os._exit(exitstatus) | |
Xet Storage Details
- Size:
- 41.9 kB
- Xet hash:
- 33387a59daabafda5ee2e22294223f4424d8d4a0d65e9eab7649fcb38d729074
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.