auto / scripts /verify_gate5_security_audit.py
rsnarsna
feat(alli-backend): Wave-1 standardization, standalone sys.env node & vault crypto, and DEC-014 security hardening
8628c98
Raw History Blame Contribute Delete
8.78 kB
"""
ALLI Phase 18: Gate 5 — Security & Isolation Audit
Verifies:
1. Credential Stripping: Sensitive environment variables (AWS_*, SUPABASE_*, MONGODB_URI, etc.) stripped from runtime
2. Air-gapped Network Isolation: Docker runner configured with --network none
3. Resource Boundaries: Memory limit (128m), CPU limit (0.5)
4. Subprocess Execution Timeout: 5.0s / configurable timeout terminates infinite loops cleanly
5. Graceful Fallback Behavior: Multi-tier fallback (Docker -> Local subprocess)
6. Ephemeral File Cleanup: Temporary execution files reliably removed from filesystem
"""
import os
import sys
import time
from pathlib import Path
# Add backend root directory to sys.path
sys.path.insert(0, str(Path(__file__).parent.parent))
from runtime_environment.runner import (
get_sanitized_env,
LocalSubprocessRunner,
DockerSandboxRunner,
RuntimeRunner,
)
def run_gate5_security_audit():
print("=" * 80)
print("ALLI PHASE 18: GATE 5 — SECURITY & ISOLATION AUDIT")
print("=" * 80)
passed = 0
total = 0
def assert_test(condition: bool, msg: str):
nonlocal passed, total
total += 1
if not condition:
print(f" [FAIL] {msg}")
raise AssertionError(f"Gate 5 Assertion Failed: {msg}")
passed += 1
print(f" [PASS] {msg}")
# =========================================================================
# 1. CREDENTIAL STRIPPING AUDIT
# =========================================================================
print("\n--- 1. Credential Stripping Audit ---")
# Inject sensitive mock variables into os.environ
mock_secrets = {
"AWS_ACCESS_KEY_ID": "AKIA_MOCK_KEY_12345",
"AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
"SUPABASE_SERVICE_ROLE_KEY": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.mock",
"MONGODB_URI": "mongodb+srv://admin:supersecret@cluster.mongodb.net",
"DATABASE_URL": "postgresql://postgres:secretpassword@localhost:5432/alli",
"JWT_SECRET": "ultra_secure_jwt_token_secret_value",
"CUSTOM_API_KEY": "alli_live_api_key_abcdef",
"AUTH_BEARER_TOKEN": "mock_bearer_token_string",
"PRIVATE_KEY_PEM": "-----BEGIN PRIVATE KEY-----MOCK-----END PRIVATE KEY-----",
}
for k, v in mock_secrets.items():
os.environ[k] = v
try:
sanitized_env = get_sanitized_env()
for k in mock_secrets:
assert_test(k not in sanitized_env, f"Sensitive secret '{k}' stripped from execution environment")
# Verify safe runtime flags are preserved
assert_test(sanitized_env.get("PYTHONUNBUFFERED") == "1", "PYTHONUNBUFFERED=1 enabled for real-time stdout")
assert_test(sanitized_env.get("PYTHONDONTWRITEBYTECODE") == "1", "PYTHONDONTWRITEBYTECODE=1 enabled")
# Execute a Python script inside LocalSubprocessRunner that attempts to read the secrets
probe_script = """
import os
secrets_found = []
for key in [
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "SUPABASE_SERVICE_ROLE_KEY",
"MONGODB_URI", "DATABASE_URL", "JWT_SECRET", "CUSTOM_API_KEY", "AUTH_BEARER_TOKEN"
]:
val = os.getenv(key)
if val is not None:
secrets_found.append(f"{key}={val}")
if secrets_found:
print("LEAK:" + ",".join(secrets_found))
else:
print("CLEAN_ENVIRONMENT")
"""
runner = LocalSubprocessRunner(timeout_sec=5.0)
res = runner.execute_code(probe_script)
assert_test(res.success is True, "Subprocess executed probe script successfully")
assert_test(res.stdout.strip() == "CLEAN_ENVIRONMENT", f"Subprocess confirmed air-gapped clean environment: {res.stdout.strip()!r}")
assert_test("LEAK" not in res.stdout, "Zero secrets leaked into running subprocess")
finally:
for k in mock_secrets:
os.environ.pop(k, None)
# =========================================================================
# 2. SUBPROCESS TIMEOUT ENFORCEMENT
# =========================================================================
print("\n--- 2. Subprocess Timeout Enforcement Audit ---")
infinite_loop_code = """
import time
print("Loop started")
while True:
time.sleep(0.05)
"""
timed_runner = LocalSubprocessRunner(timeout_sec=2.0)
start_t = time.perf_counter()
loop_res = timed_runner.execute_code(infinite_loop_code)
elapsed_sec = time.perf_counter() - start_t
assert_test(loop_res.success is False, "Infinite loop terminated with success=False")
assert_test(loop_res.exit_code == -1, "Timed out process yielded exit_code=-1")
assert_test(elapsed_sec < 3.0, f"Process bounded by timeout: elapsed={elapsed_sec:.2f}s (budget: 2.0s + 1.0s grace)")
assert_test("timed out after 2.0 seconds" in loop_res.stderr.lower(), f"Timeout diagnostic in stderr: {loop_res.stderr!r}")
# =========================================================================
# 3. AIR-GAPPED DOCKER SANDBOX SPECIFICATION
# =========================================================================
print("\n--- 3. Air-Gapped Docker Sandbox Specification Audit ---")
docker_runner = DockerSandboxRunner(
# Default manylinux-compatible image (python:3.12-slim)
memory_limit="128m",
cpu_limit="0.5",
timeout_sec=5.0
)
assert_test(docker_runner.image_name == "python:3.12-slim", "Docker default base image matches python:3.12-slim")
assert_test(docker_runner.memory_limit == "128m", "Memory limit enforced to 128MB")
assert_test(docker_runner.cpu_limit == "0.5", "CPU cap enforced to 0.5 cores")
assert_test(docker_runner.timeout_sec == 5.0, "Container timeout bounded to 5.0s")
# =========================================================================
# 4. MULTI-TIER RUNTIME DISPATCH & GRACEFUL FALLBACK
# =========================================================================
print("\n--- 4. Multi-Tier Runtime Dispatch & Graceful Fallback ---")
dispatcher = RuntimeRunner(default_mode="auto", timeout_sec=5.0)
# Auto mode executes valid code cleanly (falling back to subprocess if docker is unavailable)
test_code = "print('ALLI_DISPATCH_TEST_OK')"
disp_res = dispatcher.run(test_code)
assert_test(disp_res.success is True, "Runtime dispatcher executed code successfully")
assert_test("ALLI_DISPATCH_TEST_OK" in disp_res.stdout, "Dispatcher captured stdout correctly")
assert_test(disp_res.mode in ("local_subprocess", "docker_sandbox"), f"Valid execution mode recorded: {disp_res.mode}")
# Local mode runs explicitly
local_disp_res = dispatcher.run(test_code, mode="local")
assert_test(local_disp_res.success is True, "Explicit local mode succeeded")
assert_test(local_disp_res.mode == "local_subprocess", "Mode correctly set to 'local_subprocess'")
# Explicit docker mode throws clean error if daemon is not running
if not DockerSandboxRunner.is_docker_available():
docker_failed_cleanly = False
try:
dispatcher.run(test_code, mode="docker")
except RuntimeError as e:
if "Docker daemon is not available" in str(e) or "Docker was explicitly requested" in str(e):
docker_failed_cleanly = True
assert_test(docker_failed_cleanly, "Explicit docker mode safely rejected with actionable RuntimeError when Docker unavailable")
else:
print(" [INFO] Docker daemon is running on this machine; testing live container sandbox...")
live_docker_res = dispatcher.run("print('CONTAINER_OK')", mode="docker")
assert_test(live_docker_res.success is True, "Live Docker sandbox execution passed")
assert_test("CONTAINER_OK" in live_docker_res.stdout, "Docker stdout captured")
# =========================================================================
# 5. EPHEMERAL ARTIFACT & TEMP FILE DISPOSAL
# =========================================================================
print("\n--- 5. Ephemeral Artifact & Temp File Disposal ---")
import tempfile
temp_dir = tempfile.gettempdir()
pre_count = len([f for f in os.listdir(temp_dir) if f.startswith("alli_sandbox_") or f.endswith(".py")])
for i in range(5):
clean_res = runner.execute_code(f"print({i} * 10)")
assert_test(clean_res.success is True, f"Ephemeral run {i+1} succeeded")
post_count = len([f for f in os.listdir(temp_dir) if f.startswith("alli_sandbox_")])
assert_test(post_count == 0, "All temporary sandbox directories cleaned up with zero leakage")
print("\n" + "=" * 80)
print(f"GATE 5 VERIFICATION: ALL {passed}/{total} ASSERTIONS PASSED (100% OK)")
print("=" * 80)
if __name__ == "__main__":
run_gate5_security_audit()