Buckets:

download
raw
20.4 kB
#!/usr/bin/env python3
"""Memory benchmark for CIRIS agent under message load.
Measures RSS memory growth while submitting N async messages to the API.
Designed for fast, reliable mock LLM profiling up to 1k+ messages.
"""
import argparse
import asyncio
import json
import os
import sqlite3
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
def format_size(size: float) -> str:
"""Format bytes as human-readable string."""
for unit in ["B", "KB", "MB", "GB"]:
if abs(size) < 1024.0:
return f"{size:.1f} {unit}"
size /= 1024.0
return f"{size:.1f} TB"
def get_process_memory(pid: int) -> int:
"""Get RSS memory for a process in bytes."""
try:
import psutil
process = psutil.Process(pid)
return process.memory_info().rss
except ImportError:
try:
with open(f"/proc/{pid}/statm") as f:
pages = int(f.read().split()[1])
return pages * os.sysconf("SC_PAGE_SIZE")
except Exception:
return 0
except Exception:
return 0
def get_children_memory(pid: int) -> int:
"""Get total RSS memory for a process and all its children."""
try:
import psutil
parent = psutil.Process(pid)
total = parent.memory_info().rss
for child in parent.children(recursive=True):
try:
total += child.memory_info().rss
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
return total
except ImportError:
return get_process_memory(pid)
except Exception:
return 0
async def wait_for_server(base_url: str, timeout_seconds: int = 90) -> bool:
"""Wait for API server to become healthy."""
import httpx
deadline = time.time() + timeout_seconds
async with httpx.AsyncClient(timeout=3.0) as client:
while time.time() < deadline:
try:
response = await client.get(f"{base_url}/v1/system/health")
if response.status_code == 200:
return True
except Exception:
pass
await asyncio.sleep(1.0)
return False
async def wait_for_runtime_ready(base_url: str, token: str, timeout_seconds: int = 120) -> bool:
"""Wait for agent runtime endpoints to accept authenticated traffic."""
import httpx
deadline = time.time() + timeout_seconds
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=5.0) as client:
while time.time() < deadline:
try:
response = await client.get(f"{base_url}/v1/agent/status", headers=headers)
if response.status_code == 200:
try:
data = response.json()
except Exception:
data = {}
cognitive_state = data.get("data", {}).get("cognitive_state")
if cognitive_state:
print(f" Runtime state poll: {cognitive_state}")
return True
except Exception:
pass
await asyncio.sleep(1.0)
return False
async def get_auth_token(base_url: str, retries: int = 30) -> str:
"""Get auth token from API with retries."""
import httpx
last_status = 0
async with httpx.AsyncClient(timeout=10.0) as client:
for attempt in range(retries):
try:
for username in ["admin", "owner"]:
response = await client.post(
f"{base_url}/v1/auth/login",
json={"username": username, "password": "qa_test_password_12345"},
)
last_status = response.status_code
if response.status_code == 200:
return response.json().get("access_token", "")
except Exception:
pass
await asyncio.sleep(0.5)
if last_status:
print(f" Login failed after {retries} attempts (last status: {last_status})")
return ""
async def complete_setup(base_url: str, port: int) -> bool:
"""Complete first-run setup for benchmark environments."""
import httpx
payload = {
"llm_provider": "openai",
"llm_api_key": "test-key-for-benchmark",
"llm_model": "gpt-4",
"template_id": "default",
"enabled_adapters": ["api"],
"adapter_config": {},
"admin_username": "owner",
"admin_password": "qa_test_password_12345",
"agent_port": port,
}
try:
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.post(f"{base_url}/v1/setup/complete", json=payload)
print(f" Setup response status: {response.status_code}")
print(f" Setup response body: {response.text[:300]}")
return response.status_code == 200
except Exception as exc:
print(f" Setup request failed: {exc}")
return False
async def send_messages(
base_url: str,
messages: int,
token: str,
concurrency: int,
process_tasks: bool,
) -> tuple[int, int, float, list[str], dict[str, int]]:
"""Submit messages concurrently, optionally forcing full task processing.
Returns (success, failed, elapsed, task_ids, failure_reasons).
failure_reasons is a count of (status_code or exception_type) → count
so a CI summary can show *why* messages failed instead of just the
aggregate count.
"""
import httpx
from collections import Counter
queue: asyncio.Queue[int] = asyncio.Queue()
for i in range(messages):
queue.put_nowait(i)
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
success = 0
failed = 0
task_ids: list[str] = []
failure_reasons: Counter[str] = Counter()
counter_lock = asyncio.Lock()
start = time.time()
limits = httpx.Limits(max_keepalive_connections=max(20, concurrency * 2), max_connections=max(40, concurrency * 4))
# Read budget: process_tasks=True waits for a full DMA pipeline + SPEAK
# (~minutes under live LLM load); process_tasks=False is the fast queue
# path (just task acceptance), but 10s was too tight on GitHub Actions
# runners under 20-way concurrent burst — auth/rate-limit middleware +
# SQLite write contention on the audit + tasks tables added enough
# tail-latency to push 21% of /v1/agent/message calls past 10s, all
# surfacing as `exc_ReadTimeout`. 30s gives breathing room for the
# acceptance path while still catching genuine hangs.
client_timeout = (
httpx.Timeout(connect=5.0, read=60.0, write=30.0, pool=10.0)
if process_tasks
else httpx.Timeout(connect=5.0, read=30.0, write=10.0, pool=10.0)
)
async def worker(worker_id: int) -> None:
nonlocal success, failed
async with httpx.AsyncClient(timeout=client_timeout, limits=limits) as client:
while True:
try:
idx = queue.get_nowait()
except asyncio.QueueEmpty:
return
payload = {
"message": f"Memory benchmark message {idx + 1} of {messages}",
"context": {"channel_id": f"memory_benchmark_w{worker_id}_m{idx}"},
}
accepted = False
last_reason = "no_attempts"
for attempt in range(3):
endpoint = "/v1/agent/interact" if process_tasks else "/v1/agent/message"
try:
response = await client.post(f"{base_url}{endpoint}", json=payload, headers=headers)
if response.status_code == 200:
response_json = response.json()
if process_tasks:
accepted = True
else:
task_id = response_json.get("task_id") or response_json.get("data", {}).get("task_id")
accepted = bool(task_id)
if accepted and isinstance(task_id, str):
async with counter_lock:
task_ids.append(task_id)
if accepted:
break
last_reason = "200_no_task_id"
else:
last_reason = f"http_{response.status_code}"
except Exception as e:
last_reason = f"exc_{type(e).__name__}"
await asyncio.sleep(0.15 * (attempt + 1))
async with counter_lock:
if accepted:
success += 1
else:
failed += 1
failure_reasons[last_reason] += 1
if (idx + 1) % 100 == 0:
print(f" Submitted {idx + 1}/{messages} messages...")
queue.task_done()
workers = [asyncio.create_task(worker(i)) for i in range(max(1, concurrency))]
await asyncio.gather(*workers)
elapsed = time.time() - start
return success, failed, elapsed, task_ids, dict(failure_reasons)
def _extract_task_ids_from_payload(payload_text: str) -> set[str]:
"""Extract all task_id values from a serialized payload."""
found: set[str] = set()
try:
payload = json.loads(payload_text)
except Exception:
return found
def walk(value: object) -> None:
if isinstance(value, dict):
maybe_task_id = value.get("task_id")
if isinstance(maybe_task_id, str):
found.add(maybe_task_id)
for nested in value.values():
walk(nested)
elif isinstance(value, list):
for nested in value:
walk(nested)
walk(payload)
return found
def verify_task_processing(
task_ids: list[str],
expected_count: int,
start_time_iso: str,
timeout_seconds: int = 180,
) -> tuple[bool, int, int]:
"""Verify submitted tasks reached SPEAK and TASK_COMPLETE in local audit DB."""
submitted = set(task_ids)
deadline = time.time() + timeout_seconds
speak_count = 0
task_complete_count = 0
while time.time() < deadline:
speak_ids: set[str] = set()
complete_ids: set[str] = set()
try:
conn = sqlite3.connect("data/ciris_audit.db")
cursor = conn.cursor()
cursor.execute(
"""
SELECT event_type, event_payload
FROM audit_log
WHERE event_timestamp >= ?
AND event_type IN ('speak', 'task_complete')
""",
(start_time_iso,),
)
rows = cursor.fetchall()
conn.close()
speak_count = 0
task_complete_count = 0
for event_type, event_payload in rows:
if event_type == "speak":
speak_count += 1
elif event_type == "task_complete":
task_complete_count += 1
if isinstance(event_payload, str):
ids = _extract_task_ids_from_payload(event_payload)
if event_type == "speak":
speak_ids.update(ids)
elif event_type == "task_complete":
complete_ids.update(ids)
if submitted and submitted.issubset(speak_ids) and submitted.issubset(complete_ids):
return True, task_complete_count, speak_count
required = expected_count if expected_count > 0 else len(submitted)
if speak_count >= required and task_complete_count >= required:
return True, task_complete_count, speak_count
except Exception:
pass
time.sleep(2.0)
return False, task_complete_count, speak_count
def main(
messages: int = 100,
adapter: str = "api",
port: int = 8080,
concurrency: int = 20,
verify_audit: bool = False,
idle_seconds: int = 0,
) -> int:
"""Run memory benchmark with N messages."""
print("=" * 70)
print(f"CIRIS MEMORY BENCHMARK - {messages} MESSAGES")
print("=" * 70)
project_root = Path(__file__).parent.parent
main_py = project_root / "main.py"
base_url = f"http://localhost:{port}"
print("\n[1/4] Starting CIRIS API server...")
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
# Benchmarks measure per-task throughput, so bypass the channel-level task-append
# coalescing in BaseObserver. See ciris_engine/logic/adapters/base_observer.py.
env.setdefault("CIRIS_DISABLE_TASK_APPEND", "1")
cmd = [
sys.executable,
str(main_py),
"--adapter",
adapter,
"--mock-llm",
"--port",
str(port),
]
process = subprocess.Popen(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
cwd=str(project_root),
env=env,
)
pid = process.pid
print(f" PID: {pid}")
def shutdown_process() -> None:
if process.poll() is not None:
return
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
print("\n[2/4] Waiting for server...")
if not asyncio.run(wait_for_server(base_url)):
print(" ERROR: Server failed to start")
shutdown_process()
return 1
print(" Server ready!")
initial_mem = get_children_memory(pid)
print(f"\n Initial memory: {format_size(initial_mem)}")
print("\n[3/4] Authenticating...")
token = asyncio.run(get_auth_token(base_url))
if not token:
print(" No auth token yet, attempting first-run setup...")
setup_ok = asyncio.run(complete_setup(base_url, port))
if setup_ok:
print(" Setup completed; retrying authentication...")
token = asyncio.run(get_auth_token(base_url))
if not token:
print(" ERROR: Could not get auth token with QA credentials")
shutdown_process()
return 1
runtime_ready = asyncio.run(wait_for_runtime_ready(base_url, token))
if not runtime_ready:
print(" ERROR: Agent runtime did not become ready for authenticated traffic")
shutdown_process()
return 1
benchmark_start_time = datetime.now(timezone.utc).isoformat()
print(f"\n[4/4] Sending {messages} messages (concurrency={concurrency})...")
samples = []
success = 0
failed = 0
failure_reasons: dict[str, int] = {}
try:
success, failed, elapsed, task_ids, failure_reasons = asyncio.run(
send_messages(base_url, messages, token, concurrency, process_tasks=verify_audit)
)
print(f" Submitted {success}/{messages} messages in {elapsed:.1f}s")
if failure_reasons:
print(f" Failure breakdown ({failed} total):")
for reason, count in sorted(failure_reasons.items(), key=lambda x: -x[1]):
print(f" {count:>4}× {reason}")
if verify_audit and success > 0:
verified, task_complete_count, speak_count = verify_task_processing(task_ids, success, benchmark_start_time)
if verified:
print(
f" Audit verification: TASK_COMPLETE={task_complete_count}, SPEAK={speak_count} "
f"(expected >= {success})"
)
else:
print(
" ERROR: Not all submitted tasks reached SPEAK/TASK_COMPLETE in audit trail "
f"(TASK_COMPLETE={task_complete_count}, SPEAK={speak_count}, expected={success})"
)
failed += success
for _ in range(5):
mem = get_children_memory(pid)
if mem > 0:
samples.append(mem)
time.sleep(0.5)
# Optional idle window: hold the server alive after the burst and
# sample memory periodically so we can observe whether async task
# processing eventually drains and memory plateaus. Without this,
# the headline "Final" memory in CI is taken 2.5s after submission
# — well before the queued thoughts have actually finished
# processing — and underrepresents steady-state RSS.
if idle_seconds > 0:
print(f"\n Idle settling window: {idle_seconds}s, sampling every 2s...")
idle_samples: list[tuple[float, float]] = []
t0 = time.time()
while time.time() - t0 < idle_seconds:
t = time.time() - t0
mem = get_children_memory(pid)
if mem > 0:
samples.append(mem)
idle_samples.append((t, mem / 1024 / 1024))
time.sleep(2.0)
if idle_samples:
first = idle_samples[0][1]
last = idle_samples[-1][1]
peak = max(s[1] for s in idle_samples)
# Plateau = last 10% of samples agree within 1 MB
tail = idle_samples[-max(1, len(idle_samples) // 10):]
stable = (max(s[1] for s in tail) - min(s[1] for s in tail)) < 1.0
print(
f" Idle: first={first:.1f}MB peak={peak:.1f}MB final={last:.1f}MB "
f"delta={last - first:+.1f}MB plateau={'yes' if stable else 'no'}"
)
except Exception as e:
print(f" ERROR: {e}")
finally:
print("\nShutting down server...")
shutdown_process()
print("\n" + "=" * 70)
print("MEMORY BENCHMARK RESULTS")
print("=" * 70)
final_mem = samples[-1] if samples else 0
peak_mem = max(samples) if samples else 0
print(f" Messages: {messages}")
print(f" Initial: {format_size(initial_mem)}")
print(f" Final: {format_size(final_mem)}")
print(f" Peak: {format_size(peak_mem)}")
print(f" Growth: {format_size(final_mem - initial_mem)}")
if messages > 0 and final_mem > initial_mem:
per_msg = (final_mem - initial_mem) / messages
print(f" Per Message: {format_size(per_msg)}")
print(f"\n Adapter: {adapter}")
print(" Mock LLM: True")
print(f" Concurrency: {concurrency}")
print(f" Success Rate: {success}/{messages} ({100 * success / messages:.1f}%)")
print(f" Failures: {failed}")
print("=" * 70)
return 0 if success == messages else 1
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="CIRIS memory benchmark")
parser.add_argument(
"--messages",
type=int,
default=100,
help="Number of messages to send (default: 100)",
)
parser.add_argument(
"--adapter",
default="api",
help="Adapter to use (default: api)",
)
parser.add_argument(
"--port",
type=int,
default=8080,
help="Port for API server (default: 8080)",
)
parser.add_argument(
"--concurrency",
type=int,
default=20,
help="Number of concurrent submit workers (default: 20)",
)
parser.add_argument("--verify-audit", action="store_true", help="Verify SPEAK/TASK_COMPLETE entries in audit log")
parser.add_argument(
"--idle-seconds",
type=int,
default=0,
help=(
"After submitting all messages, hold the server alive for N seconds "
"and sample memory every 2s. Reveals whether async task processing "
"drains and RAM plateaus. Default 0 (matches old behavior)."
),
)
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
sys.exit(
main(
messages=args.messages,
adapter=args.adapter,
port=args.port,
concurrency=max(1, args.concurrency),
verify_audit=args.verify_audit,
idle_seconds=max(0, args.idle_seconds),
)
)

Xet Storage Details

Size:
20.4 kB
·
Xet hash:
6727f9eeaf33f3aa2079b8f70d621bade21afb70616d991035a7ff5dd9d8814f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.