Buckets:

download
raw
17.4 kB
#!/usr/bin/env python3
"""Introspect Python memory usage of running CIRIS agent.
Usage:
python3 tools/introspect_memory.py --adapters api --duration 30
python3 tools/introspect_memory.py --adapters api --messages 1000 --concurrency 8
"""
import argparse
import asyncio
import json
import os
import sqlite3
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
def format_size(size: float) -> str:
for unit in ["B", "KB", "MB", "GB"]:
if abs(size) < 1024.0:
return f"{size:.1f} {unit}"
size /= 1024.0
return f"{size:.1f} TB"
def get_process_memory(pid: int) -> int:
"""Get RSS memory for a process in bytes."""
try:
import psutil
process = psutil.Process(pid)
return process.memory_info().rss
except ImportError:
try:
with open(f"/proc/{pid}/statm") as f:
pages = int(f.read().split()[1])
return pages * os.sysconf("SC_PAGE_SIZE")
except Exception:
return 0
except Exception:
return 0
def get_children_memory(pid: int) -> int:
"""Get total RSS memory for a process and all its children."""
try:
import psutil
parent = psutil.Process(pid)
total = parent.memory_info().rss
for child in parent.children(recursive=True):
try:
total += child.memory_info().rss
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
return total
except ImportError:
return get_process_memory(pid)
except Exception:
return 0
async def _wait_for_server(base_url: str, timeout_seconds: int = 90) -> bool:
import httpx
deadline = time.time() + timeout_seconds
async with httpx.AsyncClient(timeout=3.0) as client:
while time.time() < deadline:
try:
response = await client.get(f"{base_url}/v1/system/health")
if response.status_code == 200:
return True
except Exception:
pass
await asyncio.sleep(1.0)
return False
async def _get_auth_token(base_url: str, retries: int = 30) -> str:
import httpx
async with httpx.AsyncClient(timeout=10.0) as client:
for _ in range(retries):
try:
for username in ["admin", "owner"]:
response = await client.post(
f"{base_url}/v1/auth/login",
json={"username": username, "password": "qa_test_password_12345"},
)
if response.status_code == 200:
return response.json().get("access_token", "")
except Exception:
pass
await asyncio.sleep(0.5)
return ""
async def _complete_setup(base_url: str, port: int) -> bool:
"""Complete first-run setup for local benchmark environments."""
import httpx
payload = {
"llm_provider": "openai",
"llm_api_key": "test-key-for-introspection",
"llm_model": "gpt-4",
"template_id": "default",
"enabled_adapters": ["api"],
"adapter_config": {},
"admin_username": "owner",
"admin_password": "qa_test_password_12345",
"agent_port": port,
}
try:
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.post(f"{base_url}/v1/setup/complete", json=payload)
print(f"Setup response status: {response.status_code}")
print(f"Setup response body: {response.text[:300]}")
return response.status_code == 200
except Exception as exc:
print(f"Setup request failed: {exc}")
return False
async def _wait_for_runtime_ready(base_url: str, token: str, timeout_seconds: int = 120) -> bool:
import httpx
deadline = time.time() + timeout_seconds
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=5.0) as client:
while time.time() < deadline:
try:
response = await client.get(f"{base_url}/v1/agent/status", headers=headers)
if response.status_code == 200:
try:
data = response.json()
except Exception:
data = {}
cognitive_state = data.get("data", {}).get("cognitive_state")
if cognitive_state:
print(f"Runtime state poll: {cognitive_state}")
return True
except Exception:
pass
await asyncio.sleep(1.0)
return False
async def _run_message_load(
base_url: str,
token: str,
messages: int,
concurrency: int,
process_tasks: bool,
) -> tuple[int, int, float, list[str]]:
import httpx
queue: asyncio.Queue[int] = asyncio.Queue()
for i in range(messages):
queue.put_nowait(i)
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
success = 0
failed = 0
task_ids: list[str] = []
lock = asyncio.Lock()
start = time.time()
client_timeout = (
httpx.Timeout(connect=5.0, read=60.0, write=30.0, pool=10.0)
if process_tasks
else httpx.Timeout(connect=5.0, read=10.0, write=10.0, pool=10.0)
)
async def worker(worker_id: int) -> None:
nonlocal success, failed
async with httpx.AsyncClient(timeout=client_timeout) as client:
while True:
try:
idx = queue.get_nowait()
except asyncio.QueueEmpty:
return
payload = {
"message": f"Introspection benchmark message {idx + 1}/{messages}",
"context": {"channel_id": f"introspection_w{worker_id}_m{idx}"},
}
try:
accepted = False
for attempt in range(3):
endpoint = "/v1/agent/interact" if process_tasks else "/v1/agent/message"
response = await client.post(f"{base_url}{endpoint}", headers=headers, json=payload)
if response.status_code == 200:
response_json = response.json()
if process_tasks:
accepted = True
else:
task_id = response_json.get("task_id") or response_json.get("data", {}).get("task_id")
accepted = bool(task_id)
if accepted and isinstance(task_id, str):
async with lock:
task_ids.append(task_id)
if accepted:
break
await asyncio.sleep(0.15 * (attempt + 1))
async with lock:
if accepted:
success += 1
else:
failed += 1
except Exception:
async with lock:
failed += 1
queue.task_done()
workers = [asyncio.create_task(worker(i)) for i in range(max(1, concurrency))]
await asyncio.gather(*workers)
return success, failed, time.time() - start, task_ids
def _extract_task_ids_from_payload(payload_text: str) -> set[str]:
found: set[str] = set()
try:
payload = json.loads(payload_text)
except Exception:
return found
def walk(value: object) -> None:
if isinstance(value, dict):
maybe_task_id = value.get("task_id")
if isinstance(maybe_task_id, str):
found.add(maybe_task_id)
for nested in value.values():
walk(nested)
elif isinstance(value, list):
for nested in value:
walk(nested)
walk(payload)
return found
def _verify_task_processing(
task_ids: list[str],
expected_count: int,
start_time_iso: str,
timeout_seconds: int = 180,
) -> tuple[bool, int, int]:
submitted = set(task_ids)
deadline = time.time() + timeout_seconds
speak_count = 0
task_complete_count = 0
while time.time() < deadline:
speak_ids: set[str] = set()
complete_ids: set[str] = set()
try:
conn = sqlite3.connect("data/ciris_audit.db")
cursor = conn.cursor()
cursor.execute(
"""
SELECT event_type, event_payload
FROM audit_log
WHERE event_timestamp >= ?
AND event_type IN ('speak', 'task_complete')
""",
(start_time_iso,),
)
rows = cursor.fetchall()
conn.close()
speak_count = 0
task_complete_count = 0
for event_type, event_payload in rows:
if event_type == "speak":
speak_count += 1
elif event_type == "task_complete":
task_complete_count += 1
if isinstance(event_payload, str):
ids = _extract_task_ids_from_payload(event_payload)
if event_type == "speak":
speak_ids.update(ids)
elif event_type == "task_complete":
complete_ids.update(ids)
if submitted and submitted.issubset(speak_ids) and submitted.issubset(complete_ids):
return True, task_complete_count, speak_count
required = expected_count if expected_count > 0 else len(submitted)
if speak_count >= required and task_complete_count >= required:
return True, task_complete_count, speak_count
except Exception:
pass
time.sleep(2.0)
return False, task_complete_count, speak_count
def main(
adapters: str = "cli",
duration: int = 30,
messages: int = 0,
concurrency: int = 20,
port: int = 8080,
verify_audit: bool = False,
) -> int:
print("=" * 70)
print("CIRIS 2.0 MEMORY INTROSPECTION")
print("=" * 70)
print(f"Adapters: {adapters}")
print(f"Duration: {duration}s")
if messages > 0:
print(f"Message load: {messages} (concurrency={concurrency})")
project_root = Path(__file__).parent.parent
main_py = project_root / "main.py"
if not main_py.exists():
print(f"ERROR: main.py not found at {main_py}")
return 1
cmd = [sys.executable, str(main_py), "--adapter", adapters, "--mock-llm"]
if adapters == "api":
cmd.extend(["--port", str(port)])
else:
cmd.extend(["--timeout", str(duration)])
print(f"\nCommand: {' '.join(cmd)}")
print("\n[1/3] Starting CIRIS agent...")
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
# Benchmarks measure per-task throughput, so bypass the channel-level task-append
# coalescing in BaseObserver. See ciris_engine/logic/adapters/base_observer.py.
env.setdefault("CIRIS_DISABLE_TASK_APPEND", "1")
process = subprocess.Popen(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
cwd=str(project_root),
env=env,
)
pid = process.pid
print(f"PID: {pid}")
def shutdown_process() -> None:
if process.poll() is not None:
return
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
samples = []
start_time = time.time()
try:
if adapters == "api" and messages > 0:
base_url = f"http://localhost:{port}"
if not asyncio.run(_wait_for_server(base_url)):
print("ERROR: API server did not become healthy")
shutdown_process()
return 1
token = asyncio.run(_get_auth_token(base_url))
if not token:
print("No auth token yet, attempting first-run setup...")
setup_ok = asyncio.run(_complete_setup(base_url, port))
if setup_ok:
print("Setup completed; retrying authentication...")
token = asyncio.run(_get_auth_token(base_url))
if not token:
print("ERROR: Could not authenticate with QA credentials")
shutdown_process()
return 1
runtime_ready = asyncio.run(_wait_for_runtime_ready(base_url, token))
if not runtime_ready:
print("ERROR: Agent runtime did not become ready for authenticated traffic")
shutdown_process()
return 1
print("\n[2/3] Running message load...")
load_start = datetime.now(timezone.utc).isoformat()
success, failed, elapsed, task_ids = asyncio.run(
_run_message_load(base_url, token, messages, concurrency, process_tasks=verify_audit)
)
print(f" Message load complete: {success}/{messages} in {elapsed:.1f}s (failed={failed})")
if verify_audit and success > 0:
verified, task_complete_count, speak_count = _verify_task_processing(task_ids, success, load_start)
if verified:
print(
f" Audit verification: TASK_COMPLETE={task_complete_count}, SPEAK={speak_count} "
f"(expected >= {success})"
)
else:
print(
" ERROR: Not all submitted tasks reached SPEAK/TASK_COMPLETE in audit trail "
f"(TASK_COMPLETE={task_complete_count}, SPEAK={speak_count}, expected={success})"
)
return 1
immediate_mem = get_children_memory(pid)
if immediate_mem > 0:
samples.append(immediate_mem)
settle_deadline = time.time() + min(10, duration)
while time.time() < settle_deadline and process.poll() is None:
mem = get_children_memory(pid)
if mem > 0:
samples.append(mem)
time.sleep(0.2)
else:
print(f"\n[2/3] Monitoring memory for ~{duration}s...")
end_time = start_time + duration
while time.time() < end_time and process.poll() is None:
mem = get_children_memory(pid)
if mem > 0:
samples.append(mem)
elapsed = time.time() - start_time
print(f" {elapsed:5.1f}s: {format_size(mem)}", end="\r")
time.sleep(0.1)
print()
print("\n[3/3] Shutting down process...")
shutdown_process()
print("\n" + "=" * 70)
print("MEMORY SUMMARY")
print("=" * 70)
if samples:
initial = samples[0]
peak = max(samples)
final = samples[-1]
avg = sum(samples) / len(samples)
print(f" Initial: {format_size(initial)}")
print(f" Peak: {format_size(peak)}")
print(f" Final: {format_size(final)}")
print(f" Average: {format_size(avg)}")
print(f" Samples: {len(samples)}")
else:
print(" No memory samples collected")
print(f"\n Adapters: {adapters}")
print(" Mock LLM: True")
if messages > 0:
print(f" Messages: {messages}")
print("\n" + "=" * 70)
print("INTROSPECTION COMPLETE")
print("=" * 70)
return 0
except KeyboardInterrupt:
print("\n\nInterrupted by user")
process.terminate()
process.wait(timeout=5)
return 1
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="CIRIS memory introspection tool")
parser.add_argument(
"--adapters",
default="cli",
help="Adapter to use (default: cli)",
)
parser.add_argument(
"--duration",
type=int,
default=30,
help="Seconds to run before shutdown (default: 30)",
)
parser.add_argument(
"--messages",
type=int,
default=0,
help="Optional message-load count (API adapter only)",
)
parser.add_argument(
"--concurrency",
type=int,
default=20,
help="Message-load concurrency (default: 20)",
)
parser.add_argument(
"--port",
type=int,
default=8080,
help="API port when adapters=api (default: 8080)",
)
parser.add_argument("--verify-audit", action="store_true", help="Verify SPEAK/TASK_COMPLETE entries in audit log")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
sys.exit(
main(
adapters=args.adapters,
duration=args.duration,
messages=max(0, args.messages),
concurrency=max(1, args.concurrency),
port=args.port,
verify_audit=args.verify_audit,
)
)

Xet Storage Details

Size:
17.4 kB
·
Xet hash:
6886baee6eb8c6657da52d3ef490b93d4db95e70ce03d90bff35fad591a7a012

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.