Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /tools /introspect_memory.py
| #!/usr/bin/env python3 | |
| """Introspect Python memory usage of running CIRIS agent. | |
| Usage: | |
| python3 tools/introspect_memory.py --adapters api --duration 30 | |
| python3 tools/introspect_memory.py --adapters api --messages 1000 --concurrency 8 | |
| """ | |
| import argparse | |
| import asyncio | |
| import json | |
| import os | |
| import sqlite3 | |
| import subprocess | |
| import sys | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| def format_size(size: float) -> str: | |
| for unit in ["B", "KB", "MB", "GB"]: | |
| if abs(size) < 1024.0: | |
| return f"{size:.1f} {unit}" | |
| size /= 1024.0 | |
| return f"{size:.1f} TB" | |
| def get_process_memory(pid: int) -> int: | |
| """Get RSS memory for a process in bytes.""" | |
| try: | |
| import psutil | |
| process = psutil.Process(pid) | |
| return process.memory_info().rss | |
| except ImportError: | |
| try: | |
| with open(f"/proc/{pid}/statm") as f: | |
| pages = int(f.read().split()[1]) | |
| return pages * os.sysconf("SC_PAGE_SIZE") | |
| except Exception: | |
| return 0 | |
| except Exception: | |
| return 0 | |
| def get_children_memory(pid: int) -> int: | |
| """Get total RSS memory for a process and all its children.""" | |
| try: | |
| import psutil | |
| parent = psutil.Process(pid) | |
| total = parent.memory_info().rss | |
| for child in parent.children(recursive=True): | |
| try: | |
| total += child.memory_info().rss | |
| except (psutil.NoSuchProcess, psutil.AccessDenied): | |
| pass | |
| return total | |
| except ImportError: | |
| return get_process_memory(pid) | |
| except Exception: | |
| return 0 | |
| async def _wait_for_server(base_url: str, timeout_seconds: int = 90) -> bool: | |
| import httpx | |
| deadline = time.time() + timeout_seconds | |
| async with httpx.AsyncClient(timeout=3.0) as client: | |
| while time.time() < deadline: | |
| try: | |
| response = await client.get(f"{base_url}/v1/system/health") | |
| if response.status_code == 200: | |
| return True | |
| except Exception: | |
| pass | |
| await asyncio.sleep(1.0) | |
| return False | |
| async def _get_auth_token(base_url: str, retries: int = 30) -> str: | |
| import httpx | |
| async with httpx.AsyncClient(timeout=10.0) as client: | |
| for _ in range(retries): | |
| try: | |
| for username in ["admin", "owner"]: | |
| response = await client.post( | |
| f"{base_url}/v1/auth/login", | |
| json={"username": username, "password": "qa_test_password_12345"}, | |
| ) | |
| if response.status_code == 200: | |
| return response.json().get("access_token", "") | |
| except Exception: | |
| pass | |
| await asyncio.sleep(0.5) | |
| return "" | |
| async def _complete_setup(base_url: str, port: int) -> bool: | |
| """Complete first-run setup for local benchmark environments.""" | |
| import httpx | |
| payload = { | |
| "llm_provider": "openai", | |
| "llm_api_key": "test-key-for-introspection", | |
| "llm_model": "gpt-4", | |
| "template_id": "default", | |
| "enabled_adapters": ["api"], | |
| "adapter_config": {}, | |
| "admin_username": "owner", | |
| "admin_password": "qa_test_password_12345", | |
| "agent_port": port, | |
| } | |
| try: | |
| async with httpx.AsyncClient(timeout=30.0) as client: | |
| response = await client.post(f"{base_url}/v1/setup/complete", json=payload) | |
| print(f"Setup response status: {response.status_code}") | |
| print(f"Setup response body: {response.text[:300]}") | |
| return response.status_code == 200 | |
| except Exception as exc: | |
| print(f"Setup request failed: {exc}") | |
| return False | |
| async def _wait_for_runtime_ready(base_url: str, token: str, timeout_seconds: int = 120) -> bool: | |
| import httpx | |
| deadline = time.time() + timeout_seconds | |
| headers = {"Authorization": f"Bearer {token}"} | |
| async with httpx.AsyncClient(timeout=5.0) as client: | |
| while time.time() < deadline: | |
| try: | |
| response = await client.get(f"{base_url}/v1/agent/status", headers=headers) | |
| if response.status_code == 200: | |
| try: | |
| data = response.json() | |
| except Exception: | |
| data = {} | |
| cognitive_state = data.get("data", {}).get("cognitive_state") | |
| if cognitive_state: | |
| print(f"Runtime state poll: {cognitive_state}") | |
| return True | |
| except Exception: | |
| pass | |
| await asyncio.sleep(1.0) | |
| return False | |
| async def _run_message_load( | |
| base_url: str, | |
| token: str, | |
| messages: int, | |
| concurrency: int, | |
| process_tasks: bool, | |
| ) -> tuple[int, int, float, list[str]]: | |
| import httpx | |
| queue: asyncio.Queue[int] = asyncio.Queue() | |
| for i in range(messages): | |
| queue.put_nowait(i) | |
| headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} | |
| success = 0 | |
| failed = 0 | |
| task_ids: list[str] = [] | |
| lock = asyncio.Lock() | |
| start = time.time() | |
| client_timeout = ( | |
| httpx.Timeout(connect=5.0, read=60.0, write=30.0, pool=10.0) | |
| if process_tasks | |
| else httpx.Timeout(connect=5.0, read=10.0, write=10.0, pool=10.0) | |
| ) | |
| async def worker(worker_id: int) -> None: | |
| nonlocal success, failed | |
| async with httpx.AsyncClient(timeout=client_timeout) as client: | |
| while True: | |
| try: | |
| idx = queue.get_nowait() | |
| except asyncio.QueueEmpty: | |
| return | |
| payload = { | |
| "message": f"Introspection benchmark message {idx + 1}/{messages}", | |
| "context": {"channel_id": f"introspection_w{worker_id}_m{idx}"}, | |
| } | |
| try: | |
| accepted = False | |
| for attempt in range(3): | |
| endpoint = "/v1/agent/interact" if process_tasks else "/v1/agent/message" | |
| response = await client.post(f"{base_url}{endpoint}", headers=headers, json=payload) | |
| if response.status_code == 200: | |
| response_json = response.json() | |
| if process_tasks: | |
| accepted = True | |
| else: | |
| task_id = response_json.get("task_id") or response_json.get("data", {}).get("task_id") | |
| accepted = bool(task_id) | |
| if accepted and isinstance(task_id, str): | |
| async with lock: | |
| task_ids.append(task_id) | |
| if accepted: | |
| break | |
| await asyncio.sleep(0.15 * (attempt + 1)) | |
| async with lock: | |
| if accepted: | |
| success += 1 | |
| else: | |
| failed += 1 | |
| except Exception: | |
| async with lock: | |
| failed += 1 | |
| queue.task_done() | |
| workers = [asyncio.create_task(worker(i)) for i in range(max(1, concurrency))] | |
| await asyncio.gather(*workers) | |
| return success, failed, time.time() - start, task_ids | |
| def _extract_task_ids_from_payload(payload_text: str) -> set[str]: | |
| found: set[str] = set() | |
| try: | |
| payload = json.loads(payload_text) | |
| except Exception: | |
| return found | |
| def walk(value: object) -> None: | |
| if isinstance(value, dict): | |
| maybe_task_id = value.get("task_id") | |
| if isinstance(maybe_task_id, str): | |
| found.add(maybe_task_id) | |
| for nested in value.values(): | |
| walk(nested) | |
| elif isinstance(value, list): | |
| for nested in value: | |
| walk(nested) | |
| walk(payload) | |
| return found | |
| def _verify_task_processing( | |
| task_ids: list[str], | |
| expected_count: int, | |
| start_time_iso: str, | |
| timeout_seconds: int = 180, | |
| ) -> tuple[bool, int, int]: | |
| submitted = set(task_ids) | |
| deadline = time.time() + timeout_seconds | |
| speak_count = 0 | |
| task_complete_count = 0 | |
| while time.time() < deadline: | |
| speak_ids: set[str] = set() | |
| complete_ids: set[str] = set() | |
| try: | |
| conn = sqlite3.connect("data/ciris_audit.db") | |
| cursor = conn.cursor() | |
| cursor.execute( | |
| """ | |
| SELECT event_type, event_payload | |
| FROM audit_log | |
| WHERE event_timestamp >= ? | |
| AND event_type IN ('speak', 'task_complete') | |
| """, | |
| (start_time_iso,), | |
| ) | |
| rows = cursor.fetchall() | |
| conn.close() | |
| speak_count = 0 | |
| task_complete_count = 0 | |
| for event_type, event_payload in rows: | |
| if event_type == "speak": | |
| speak_count += 1 | |
| elif event_type == "task_complete": | |
| task_complete_count += 1 | |
| if isinstance(event_payload, str): | |
| ids = _extract_task_ids_from_payload(event_payload) | |
| if event_type == "speak": | |
| speak_ids.update(ids) | |
| elif event_type == "task_complete": | |
| complete_ids.update(ids) | |
| if submitted and submitted.issubset(speak_ids) and submitted.issubset(complete_ids): | |
| return True, task_complete_count, speak_count | |
| required = expected_count if expected_count > 0 else len(submitted) | |
| if speak_count >= required and task_complete_count >= required: | |
| return True, task_complete_count, speak_count | |
| except Exception: | |
| pass | |
| time.sleep(2.0) | |
| return False, task_complete_count, speak_count | |
| def main( | |
| adapters: str = "cli", | |
| duration: int = 30, | |
| messages: int = 0, | |
| concurrency: int = 20, | |
| port: int = 8080, | |
| verify_audit: bool = False, | |
| ) -> int: | |
| print("=" * 70) | |
| print("CIRIS 2.0 MEMORY INTROSPECTION") | |
| print("=" * 70) | |
| print(f"Adapters: {adapters}") | |
| print(f"Duration: {duration}s") | |
| if messages > 0: | |
| print(f"Message load: {messages} (concurrency={concurrency})") | |
| project_root = Path(__file__).parent.parent | |
| main_py = project_root / "main.py" | |
| if not main_py.exists(): | |
| print(f"ERROR: main.py not found at {main_py}") | |
| return 1 | |
| cmd = [sys.executable, str(main_py), "--adapter", adapters, "--mock-llm"] | |
| if adapters == "api": | |
| cmd.extend(["--port", str(port)]) | |
| else: | |
| cmd.extend(["--timeout", str(duration)]) | |
| print(f"\nCommand: {' '.join(cmd)}") | |
| print("\n[1/3] Starting CIRIS agent...") | |
| env = os.environ.copy() | |
| env["PYTHONUNBUFFERED"] = "1" | |
| # Benchmarks measure per-task throughput, so bypass the channel-level task-append | |
| # coalescing in BaseObserver. See ciris_engine/logic/adapters/base_observer.py. | |
| env.setdefault("CIRIS_DISABLE_TASK_APPEND", "1") | |
| process = subprocess.Popen( | |
| cmd, | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL, | |
| cwd=str(project_root), | |
| env=env, | |
| ) | |
| pid = process.pid | |
| print(f"PID: {pid}") | |
| def shutdown_process() -> None: | |
| if process.poll() is not None: | |
| return | |
| process.terminate() | |
| try: | |
| process.wait(timeout=10) | |
| except subprocess.TimeoutExpired: | |
| process.kill() | |
| process.wait(timeout=5) | |
| samples = [] | |
| start_time = time.time() | |
| try: | |
| if adapters == "api" and messages > 0: | |
| base_url = f"http://localhost:{port}" | |
| if not asyncio.run(_wait_for_server(base_url)): | |
| print("ERROR: API server did not become healthy") | |
| shutdown_process() | |
| return 1 | |
| token = asyncio.run(_get_auth_token(base_url)) | |
| if not token: | |
| print("No auth token yet, attempting first-run setup...") | |
| setup_ok = asyncio.run(_complete_setup(base_url, port)) | |
| if setup_ok: | |
| print("Setup completed; retrying authentication...") | |
| token = asyncio.run(_get_auth_token(base_url)) | |
| if not token: | |
| print("ERROR: Could not authenticate with QA credentials") | |
| shutdown_process() | |
| return 1 | |
| runtime_ready = asyncio.run(_wait_for_runtime_ready(base_url, token)) | |
| if not runtime_ready: | |
| print("ERROR: Agent runtime did not become ready for authenticated traffic") | |
| shutdown_process() | |
| return 1 | |
| print("\n[2/3] Running message load...") | |
| load_start = datetime.now(timezone.utc).isoformat() | |
| success, failed, elapsed, task_ids = asyncio.run( | |
| _run_message_load(base_url, token, messages, concurrency, process_tasks=verify_audit) | |
| ) | |
| print(f" Message load complete: {success}/{messages} in {elapsed:.1f}s (failed={failed})") | |
| if verify_audit and success > 0: | |
| verified, task_complete_count, speak_count = _verify_task_processing(task_ids, success, load_start) | |
| if verified: | |
| print( | |
| f" Audit verification: TASK_COMPLETE={task_complete_count}, SPEAK={speak_count} " | |
| f"(expected >= {success})" | |
| ) | |
| else: | |
| print( | |
| " ERROR: Not all submitted tasks reached SPEAK/TASK_COMPLETE in audit trail " | |
| f"(TASK_COMPLETE={task_complete_count}, SPEAK={speak_count}, expected={success})" | |
| ) | |
| return 1 | |
| immediate_mem = get_children_memory(pid) | |
| if immediate_mem > 0: | |
| samples.append(immediate_mem) | |
| settle_deadline = time.time() + min(10, duration) | |
| while time.time() < settle_deadline and process.poll() is None: | |
| mem = get_children_memory(pid) | |
| if mem > 0: | |
| samples.append(mem) | |
| time.sleep(0.2) | |
| else: | |
| print(f"\n[2/3] Monitoring memory for ~{duration}s...") | |
| end_time = start_time + duration | |
| while time.time() < end_time and process.poll() is None: | |
| mem = get_children_memory(pid) | |
| if mem > 0: | |
| samples.append(mem) | |
| elapsed = time.time() - start_time | |
| print(f" {elapsed:5.1f}s: {format_size(mem)}", end="\r") | |
| time.sleep(0.1) | |
| print() | |
| print("\n[3/3] Shutting down process...") | |
| shutdown_process() | |
| print("\n" + "=" * 70) | |
| print("MEMORY SUMMARY") | |
| print("=" * 70) | |
| if samples: | |
| initial = samples[0] | |
| peak = max(samples) | |
| final = samples[-1] | |
| avg = sum(samples) / len(samples) | |
| print(f" Initial: {format_size(initial)}") | |
| print(f" Peak: {format_size(peak)}") | |
| print(f" Final: {format_size(final)}") | |
| print(f" Average: {format_size(avg)}") | |
| print(f" Samples: {len(samples)}") | |
| else: | |
| print(" No memory samples collected") | |
| print(f"\n Adapters: {adapters}") | |
| print(" Mock LLM: True") | |
| if messages > 0: | |
| print(f" Messages: {messages}") | |
| print("\n" + "=" * 70) | |
| print("INTROSPECTION COMPLETE") | |
| print("=" * 70) | |
| return 0 | |
| except KeyboardInterrupt: | |
| print("\n\nInterrupted by user") | |
| process.terminate() | |
| process.wait(timeout=5) | |
| return 1 | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="CIRIS memory introspection tool") | |
| parser.add_argument( | |
| "--adapters", | |
| default="cli", | |
| help="Adapter to use (default: cli)", | |
| ) | |
| parser.add_argument( | |
| "--duration", | |
| type=int, | |
| default=30, | |
| help="Seconds to run before shutdown (default: 30)", | |
| ) | |
| parser.add_argument( | |
| "--messages", | |
| type=int, | |
| default=0, | |
| help="Optional message-load count (API adapter only)", | |
| ) | |
| parser.add_argument( | |
| "--concurrency", | |
| type=int, | |
| default=20, | |
| help="Message-load concurrency (default: 20)", | |
| ) | |
| parser.add_argument( | |
| "--port", | |
| type=int, | |
| default=8080, | |
| help="API port when adapters=api (default: 8080)", | |
| ) | |
| parser.add_argument("--verify-audit", action="store_true", help="Verify SPEAK/TASK_COMPLETE entries in audit log") | |
| return parser.parse_args() | |
| if __name__ == "__main__": | |
| args = parse_args() | |
| sys.exit( | |
| main( | |
| adapters=args.adapters, | |
| duration=args.duration, | |
| messages=max(0, args.messages), | |
| concurrency=max(1, args.concurrency), | |
| port=args.port, | |
| verify_audit=args.verify_audit, | |
| ) | |
| ) | |
Xet Storage Details
- Size:
- 17.4 kB
- Xet hash:
- 6886baee6eb8c6657da52d3ef490b93d4db95e70ce03d90bff35fad591a7a012
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.