Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /scripts /verify.py
#!/usr/bin/env python3
"""
effGen Framework Verification Script
This validation script verifies that every component of the effGen framework
is installed, configured, and functional.
Usage:
python verify_framework.py [--verbose] [--skip-optional]
"""
import importlib
import os
import sys
import time
import traceback
from dataclasses import dataclass
from enum import Enum
# Color codes for terminal output
class Colors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
class TestStatus(Enum):
"""Status of individual tests"""
PASSED = "✓ PASSED"
FAILED = "✗ FAILED"
SKIPPED = "⊘ SKIPPED"
WARNING = "⚠ WARNING"
@dataclass
class TestResult:
"""Result of a single test"""
name: str
status: TestStatus
message: str
details: str | None = None
duration: float = 0.0
class FrameworkVerifier:
"""Main verification class for effGen framework"""
def __init__(self, verbose: bool = False, skip_optional: bool = False):
self.verbose = verbose
self.skip_optional = skip_optional
self.results: list[TestResult] = []
self.start_time = time.time()
def log(self, message: str, color: str = ""):
"""Print colored log message"""
if color:
print(f"{color}{message}{Colors.ENDC}")
else:
print(message)
def log_verbose(self, message: str):
"""Print message only in verbose mode"""
if self.verbose:
print(f" {Colors.OKCYAN}{message}{Colors.ENDC}")
def add_result(self, result: TestResult):
"""Add test result to results list"""
self.results.append(result)
# Print result
color = {
TestStatus.PASSED: Colors.OKGREEN,
TestStatus.FAILED: Colors.FAIL,
TestStatus.SKIPPED: Colors.WARNING,
TestStatus.WARNING: Colors.WARNING
}[result.status]
status_str = f"{color}{result.status.value}{Colors.ENDC}"
print(f"{status_str} {result.name} ({result.duration:.3f}s)")
if result.message and (self.verbose or result.status != TestStatus.PASSED):
print(f" → {result.message}")
if result.details and self.verbose:
print(f" Details: {result.details}")
def run_test(self, name: str, test_func, *args, **kwargs) -> TestResult:
"""Run a single test and capture result"""
start = time.time()
try:
self.log_verbose(f"Running: {name}")
test_func(*args, **kwargs)
duration = time.time() - start
return TestResult(name, TestStatus.PASSED, "Success", duration=duration)
except Exception as e:
duration = time.time() - start
error_msg = str(e)
details = traceback.format_exc() if self.verbose else None
return TestResult(name, TestStatus.FAILED, error_msg, details, duration)
def verify_imports(self):
"""Verify all core modules can be imported"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Module Imports ==={Colors.ENDC}\n")
modules = [
# Core modules
("effgen", "Main package"),
("effgen.core", "Core components"),
("effgen.core.agent", "Agent class"),
("effgen.core.task", "Task management"),
("effgen.core.state", "State management"),
("effgen.core.orchestrator", "Orchestrator"),
("effgen.core.router", "Router"),
("effgen.core.complexity_analyzer", "Complexity analyzer"),
("effgen.core.decomposition_engine", "Decomposition engine"),
("effgen.core.execution_tracker", "Execution tracker"),
("effgen.core.sub_agent_manager", "Sub-agent manager"),
# Model modules
("effgen.models", "Model system"),
("effgen.models.base", "Base model"),
("effgen.models.vllm_engine", "vLLM engine"),
("effgen.models.transformers_engine", "Transformers engine"),
("effgen.models.openai_adapter", "OpenAI adapter"),
("effgen.models.anthropic_adapter", "Anthropic adapter"),
("effgen.models.gemini_adapter", "Gemini adapter"),
("effgen.models.model_loader", "Model loader"),
# Tool modules
("effgen.tools", "Tool system"),
("effgen.tools.base_tool", "Base tool"),
("effgen.tools.registry", "Tool registry"),
("effgen.tools.builtin.calculator", "Calculator tool"),
("effgen.tools.builtin.web_search", "Web search tool"),
("effgen.tools.builtin.code_executor", "Code executor tool"),
("effgen.tools.builtin.file_ops", "File operations tool"),
("effgen.tools.builtin.python_repl", "Python REPL tool"),
# Protocol modules
("effgen.tools.protocols.mcp", "MCP protocol"),
("effgen.tools.protocols.a2a", "A2A protocol"),
("effgen.tools.protocols.acp", "ACP protocol"),
# Configuration modules
("effgen.config", "Configuration system"),
("effgen.config.loader", "Config loader"),
("effgen.config.validator", "Config validator"),
# Prompt modules
("effgen.prompts", "Prompt system"),
("effgen.prompts.template_manager", "Template manager"),
("effgen.prompts.chain_manager", "Chain manager"),
("effgen.prompts.optimizer", "Prompt optimizer"),
# Memory modules
("effgen.memory", "Memory system"),
("effgen.memory.short_term", "Short-term memory"),
("effgen.memory.long_term", "Long-term memory"),
# Execution modules
("effgen.execution", "Execution system"),
("effgen.execution.sandbox", "Code executor"),
("effgen.execution.validators", "Code validator"),
# GPU modules
("effgen.gpu", "GPU management"),
("effgen.gpu.allocator", "GPU allocator"),
("effgen.gpu.monitor", "GPU monitor"),
]
for module_name, description in modules:
def test_import(name=module_name, desc=description):
importlib.import_module(name)
self.log_verbose(f" Imported: {name}")
result = self.run_test(f"Import {description}", test_import)
self.add_result(result)
def verify_core_classes(self):
"""Verify core classes can be instantiated"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Core Class Instantiation ==={Colors.ENDC}\n")
def test_task_class():
from effgen.core.task import Task, TaskPriority, TaskStatus
task = Task(
description="Test task",
priority=TaskPriority.MEDIUM
)
assert task.description == "Test task"
assert task.priority == TaskPriority.MEDIUM
assert task.status == TaskStatus.PENDING
self.log_verbose(" Task class instantiated successfully")
def test_agent_state():
from effgen.core.state import AgentState
state = AgentState(agent_id="test_agent")
assert state.agent_id == "test_agent"
assert len(state.conversation_history) == 0
self.log_verbose(" AgentState instantiated successfully")
def test_generation_config():
from effgen.models import GenerationConfig
config = GenerationConfig(
max_tokens=100,
temperature=0.7,
top_p=0.9
)
assert config.max_tokens == 100
assert config.temperature == 0.7
self.log_verbose(" GenerationConfig instantiated successfully")
def test_tool_metadata():
from effgen.tools.base_tool import ToolCategory, ToolMetadata
metadata = ToolMetadata(
name="test_tool",
description="Test tool",
category=ToolCategory.COMPUTATION
)
assert metadata.name == "test_tool"
self.log_verbose(" ToolMetadata instantiated successfully")
def test_memory_message():
from effgen.memory import Message, MessageRole
message = Message(
role=MessageRole.USER,
content="Test message"
)
assert message.role == MessageRole.USER
assert message.content == "Test message"
self.log_verbose(" Message instantiated successfully")
tests = [
("Task class", test_task_class),
("AgentState class", test_agent_state),
("GenerationConfig class", test_generation_config),
("ToolMetadata class", test_tool_metadata),
("Memory Message class", test_memory_message),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_tool_system(self):
"""Verify tool system functionality"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Tool System ==={Colors.ENDC}\n")
def test_tool_registry():
from effgen.tools import get_registry
registry = get_registry()
assert registry is not None
self.log_verbose(" Tool registry initialized")
def test_builtin_tools():
from effgen.tools.builtin.calculator import Calculator
from effgen.tools.builtin.file_ops import FileOperations
from effgen.tools.builtin.python_repl import PythonREPL
from effgen.tools.builtin.web_search import WebSearch
# Test calculator
calc = Calculator()
assert calc.metadata.name == "calculator"
self.log_verbose(" Calculator tool initialized")
# Test web search
search = WebSearch()
assert search.metadata.name == "web_search"
self.log_verbose(" Web search tool initialized")
# Test file ops
file_ops = FileOperations()
assert file_ops.metadata.name == "file_operations"
self.log_verbose(" File operations tool initialized")
# Test Python REPL
repl = PythonREPL()
assert repl.metadata.name == "python_repl"
self.log_verbose(" Python REPL tool initialized")
def test_tool_registration():
from effgen.tools import BaseTool, ToolCategory, ToolMetadata, get_registry
from effgen.tools.base_tool import ParameterSpec, ParameterType
class TestTool(BaseTool):
def __init__(self):
super().__init__(
metadata=ToolMetadata(
name="test_verification_tool",
description="Test tool for verification",
category=ToolCategory.COMPUTATION,
parameters=[
ParameterSpec(
name="input",
type=ParameterType.STRING,
description="Test input",
required=True
)
]
)
)
async def _execute(self, input: str, **kwargs):
return f"Processed: {input}"
registry = get_registry()
registry.register_tool(TestTool)
# Verify registration
assert registry.is_registered("test_verification_tool")
self.log_verbose(" Custom tool registered successfully")
tests = [
("Tool registry initialization", test_tool_registry),
("Built-in tools", test_builtin_tools),
("Tool registration", test_tool_registration),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_configuration(self):
"""Verify configuration system"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Configuration System ==={Colors.ENDC}\n")
def test_config_loader():
from effgen.config import Config
# Test dict config
config_dict = {
"agent": {
"name": "TestAgent",
"model": {
"name": "test-model",
"engine": "transformers"
}
}
}
config = Config(data=config_dict)
assert config.agent.name == "TestAgent"
self.log_verbose(" Config loaded from dict successfully")
def test_config_validation():
from effgen.config import ConfigValidator
validator = ConfigValidator()
# ConfigValidator exists and can be instantiated
assert validator is not None
self.log_verbose(" Config validator instantiated successfully")
tests = [
("Config loader", test_config_loader),
("Config validation", test_config_validation),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_model_interfaces(self):
"""Verify model interfaces (without loading actual models)"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Model Interfaces ==={Colors.ENDC}\n")
def test_generation_config():
from effgen.models import GenerationConfig
config = GenerationConfig(
max_tokens=512,
temperature=0.7,
top_p=0.95,
top_k=50,
repetition_penalty=1.1
)
assert config.max_tokens == 512
assert config.temperature == 0.7
self.log_verbose(" GenerationConfig created successfully")
def test_model_loader_class():
from effgen.models import ModelLoader
loader = ModelLoader()
assert loader is not None
self.log_verbose(" ModelLoader class instantiated")
tests = [
("Generation config", test_generation_config),
("Model loader class", test_model_loader_class),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_prompt_system(self):
"""Verify prompt engineering system"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Prompt System ==={Colors.ENDC}\n")
def test_template_manager():
from effgen.prompts import TemplateManager
from effgen.prompts.template_manager import PromptTemplate
manager = TemplateManager()
# Create template
template = PromptTemplate(
name="test_template",
template="Hello {{ name }}, you are {{ age }} years old.",
variables=["name", "age"]
)
manager.add_template(template)
# Render template
result = template.render(name="Alice", age=30)
assert "Alice" in result
assert "30" in result
self.log_verbose(" Template created and rendered successfully")
def test_prompt_optimizer():
from effgen.prompts import PromptOptimizer
optimizer = PromptOptimizer()
# Test optimization
long_prompt = "This is a test. " * 100
result = optimizer.optimize(long_prompt, preserve_format=False)
assert len(result.optimized_prompt) <= len(long_prompt)
self.log_verbose(" Prompt optimization working")
tests = [
("Template manager", test_template_manager),
("Prompt optimizer", test_prompt_optimizer),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_memory_system(self):
"""Verify memory system"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Memory System ==={Colors.ENDC}\n")
def test_short_term_memory():
from effgen.memory import MessageRole, ShortTermMemory
memory = ShortTermMemory(max_messages=10)
# Add messages
memory.add_message(MessageRole.USER, "Hello")
memory.add_message(MessageRole.ASSISTANT, "Hi there")
messages = memory.get_recent_messages()
assert len(messages) == 2
assert messages[0].content == "Hello"
self.log_verbose(" Short-term memory working correctly")
def test_memory_entry():
from effgen.memory import ImportanceLevel, MemoryEntry, MemoryType
entry = MemoryEntry(
content="Test memory",
memory_type=MemoryType.FACT,
importance=ImportanceLevel.MEDIUM
)
assert entry.content == "Test memory"
assert entry.memory_type == MemoryType.FACT
self.log_verbose(" MemoryEntry created successfully")
tests = [
("Short-term memory", test_short_term_memory),
("Memory entry", test_memory_entry),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_execution_system(self):
"""Verify execution and validation system"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Execution System ==={Colors.ENDC}\n")
def test_code_validator():
from effgen.execution.validators import PythonValidator
validator = PythonValidator()
# Test valid code
valid_code = "x = 1 + 1\nprint(x)"
result = validator.validate(valid_code)
assert result.language == "python"
self.log_verbose(f" Code validation performed (safe: {result.is_safe})")
def test_sandbox_config():
from effgen.execution import SandboxConfig
config = SandboxConfig(
timeout=30,
memory_limit="512M",
allow_network=False
)
assert config.timeout == 30
assert config.memory_limit == "512M"
self.log_verbose(" SandboxConfig created successfully")
tests = [
("Code validator", test_code_validator),
("Sandbox config", test_sandbox_config),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_dependencies(self):
"""Verify all required dependencies are installed"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Dependencies ==={Colors.ENDC}\n")
required_deps = [
("torch", "PyTorch"),
("transformers", "Transformers"),
("yaml", "PyYAML"),
("jsonschema", "JSON Schema"),
("requests", "Requests"),
("pydantic", "Pydantic"),
("jinja2", "Jinja2"),
]
optional_deps = [
("vllm", "vLLM (for fast inference)"),
("docker", "Docker SDK"),
("anthropic", "Anthropic API"),
("openai", "OpenAI API"),
]
for module_name, description in required_deps:
def test_import(name=module_name, desc=description):
importlib.import_module(name)
self.log_verbose(f" {desc} installed")
result = self.run_test(f"Dependency: {description}", test_import)
self.add_result(result)
if not self.skip_optional:
for module_name, description in optional_deps:
def test_import(name=module_name, desc=description):
importlib.import_module(name)
self.log_verbose(f" {desc} installed")
result = self.run_test(f"Optional: {description}", test_import)
# Don't fail on optional dependencies
if result.status == TestStatus.FAILED:
result = TestResult(
result.name,
TestStatus.SKIPPED,
"Optional dependency not installed",
duration=result.duration
)
self.add_result(result)
def generate_report(self):
"""Generate final validation report"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}{'='*60}{Colors.ENDC}")
self.log(f"{Colors.HEADER}{Colors.BOLD}VERIFICATION REPORT{Colors.ENDC}")
self.log(f"{Colors.HEADER}{Colors.BOLD}{'='*60}{Colors.ENDC}\n")
# Count results
passed = sum(1 for r in self.results if r.status == TestStatus.PASSED)
failed = sum(1 for r in self.results if r.status == TestStatus.FAILED)
skipped = sum(1 for r in self.results if r.status == TestStatus.SKIPPED)
warnings = sum(1 for r in self.results if r.status == TestStatus.WARNING)
total = len(self.results)
# Calculate total time
total_time = time.time() - self.start_time
# Print summary
self.log(f"{Colors.BOLD}Summary:{Colors.ENDC}")
self.log(f" Total Tests: {total}")
self.log(f" {Colors.OKGREEN}Passed: {passed}{Colors.ENDC}")
if failed > 0:
self.log(f" {Colors.FAIL}Failed: {failed}{Colors.ENDC}")
else:
self.log(f" Failed: {failed}")
if skipped > 0:
self.log(f" {Colors.WARNING}Skipped: {skipped}{Colors.ENDC}")
if warnings > 0:
self.log(f" {Colors.WARNING}Warnings: {warnings}{Colors.ENDC}")
self.log(f" Duration: {total_time:.2f}s")
# Success rate
if total > 0:
success_rate = (passed / total) * 100
self.log(f"\n Success Rate: {success_rate:.1f}%")
# Print failures if any
if failed > 0:
self.log(f"\n{Colors.FAIL}{Colors.BOLD}Failed Tests:{Colors.ENDC}")
for result in self.results:
if result.status == TestStatus.FAILED:
self.log(f" {Colors.FAIL}✗{Colors.ENDC} {result.name}")
self.log(f" Error: {result.message}")
# Final status
self.log(f"\n{Colors.HEADER}{Colors.BOLD}{'='*60}{Colors.ENDC}")
if failed == 0:
self.log(f"{Colors.OKGREEN}{Colors.BOLD}✓ FRAMEWORK VERIFICATION PASSED{Colors.ENDC}")
self.log(f"{Colors.OKGREEN}All critical tests passed successfully!{Colors.ENDC}")
else:
self.log(f"{Colors.FAIL}{Colors.BOLD}✗ FRAMEWORK VERIFICATION FAILED{Colors.ENDC}")
self.log(f"{Colors.FAIL}{failed} test(s) failed. Please review the errors above.{Colors.ENDC}")
self.log(f"{Colors.HEADER}{Colors.BOLD}{'='*60}{Colors.ENDC}\n")
return failed == 0
def verify_agent_integration(self):
"""Verify Agent class integration with models and tools"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Agent Integration ==={Colors.ENDC}\n")
def test_agent_creation():
from effgen.core.agent import AgentConfig
# AgentConfig requires a model - just test it exists
assert AgentConfig is not None
self.log_verbose(" AgentConfig class available")
def test_task_creation():
from effgen import Task, TaskPriority
task = Task(
description="Test task for verification",
priority=TaskPriority.HIGH
)
assert task.description == "Test task for verification"
assert task.priority == TaskPriority.HIGH
self.log_verbose(" Task created successfully")
def test_tool_discovery():
from effgen import get_tool_registry
registry = get_tool_registry()
registry.discover_builtin_tools()
# Check if built-in tools were discovered
assert registry.is_registered("calculator")
self.log_verbose(" Tool discovery working")
tests = [
("Agent config creation", test_agent_creation),
("Task creation and management", test_task_creation),
("Tool discovery system", test_tool_discovery),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_protocol_systems(self):
"""Verify protocol implementations (MCP, A2A, ACP)"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Protocol Systems ==={Colors.ENDC}\n")
def test_official_mcp_import():
# Test that official MCP package is available
import mcp
assert mcp is not None
self.log_verbose(" Official MCP package imported")
def test_mcp_protocol_handler():
from effgen.tools.protocols.mcp import MCPProtocolHandler
handler = MCPProtocolHandler()
assert handler is not None
self.log_verbose(" MCP protocol handler created")
def test_mcp_client():
from effgen.tools.protocols.mcp import MCPServerConfig, TransportType
# Create a client configuration
config = MCPServerConfig(
name="test_server",
transport=TransportType.STDIO,
command="echo",
args=["test"]
)
assert config is not None
self.log_verbose(" MCP client configuration created")
def test_mcp_transports():
from effgen.tools.protocols.mcp import HTTPTransport, SSETransport, StdioTransport
# Verify all transport types are available
assert StdioTransport is not None
assert HTTPTransport is not None
assert SSETransport is not None
self.log_verbose(" MCP transport types available")
def test_mcp_server():
from effgen.tools.protocols.mcp import MCPServer, create_server
# Verify server classes exist
assert MCPServer is not None
assert create_server is not None
self.log_verbose(" MCP server classes available")
def test_mcp_data_structures():
from effgen.tools.protocols.mcp import (
MCPCapabilities,
MCPError,
MCPRequest,
MCPResource,
MCPResponse,
MCPTool,
)
# Verify all MCP data structures are available
assert MCPTool is not None
assert MCPResource is not None
assert MCPCapabilities is not None
assert MCPRequest is not None
assert MCPResponse is not None
assert MCPError is not None
self.log_verbose(" MCP data structures available")
def test_a2a_protocol():
from effgen.tools.protocols.a2a import A2AClient
# Just verify class can be imported
assert A2AClient is not None
self.log_verbose(" A2A protocol classes available")
def test_acp_protocol():
from effgen.tools.protocols.acp import ACPClient
# Just verify class can be imported
assert ACPClient is not None
self.log_verbose(" ACP protocol classes available")
def test_mcp_official_server():
from effgen.tools.protocols.mcp_official import (
EffGenMCPServer,
EffGenMCPServerConfig,
create_server,
main_http,
main_stdio,
)
# Verify all server components exist
assert EffGenMCPServer is not None
assert EffGenMCPServerConfig is not None
assert create_server is not None
assert main_stdio is not None
assert main_http is not None
self.log_verbose(" Official MCP server components available")
def test_mcp_official_client():
from effgen.tools.protocols.mcp_official import (
EffGenMCPClient,
MCPServerConfig,
create_client,
create_http_client,
create_stdio_client,
)
# Verify all client components exist
assert EffGenMCPClient is not None
assert MCPServerConfig is not None
assert create_client is not None
assert create_stdio_client is not None
assert create_http_client is not None
self.log_verbose(" Official MCP client components available")
def test_mcp_official_config():
from effgen.tools.protocols.mcp_official import MCPServerConfig
# Create configuration for STDIO transport
config = MCPServerConfig(
name="test-server",
transport="stdio",
command="python",
args=["-m", "mcp_server"]
)
assert config.name == "test-server"
assert config.transport == "stdio"
self.log_verbose(" Official MCP config creation working")
def test_mcp_official_imports():
# Test that official MCP SDK types can be imported through our wrapper
try:
from effgen.tools.protocols.mcp_official import EffGenMCPServer
# If we can import the server, official MCP is available
assert EffGenMCPServer is not None
self.log_verbose(" Official MCP SDK integration working")
except ImportError as e:
# This is okay if mcp package isn't installed
if "mcp" in str(e).lower():
self.log_verbose(" Official MCP SDK not installed (optional)")
else:
raise
tests = [
("Official MCP package", test_official_mcp_import),
("MCP protocol handler", test_mcp_protocol_handler),
("MCP client configuration", test_mcp_client),
("MCP transport types", test_mcp_transports),
("MCP server classes", test_mcp_server),
("MCP data structures", test_mcp_data_structures),
("Official MCP server components", test_mcp_official_server),
("Official MCP client components", test_mcp_official_client),
("Official MCP config creation", test_mcp_official_config),
("Official MCP SDK integration", test_mcp_official_imports),
("A2A protocol support", test_a2a_protocol),
("ACP protocol support", test_acp_protocol),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_tool_functionality(self):
"""Verify individual tool functionality"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Tool Functionality ==={Colors.ENDC}\n")
def test_calculator_execution():
import asyncio
from effgen.tools.builtin.calculator import Calculator
calc = Calculator()
# Test basic arithmetic
result = asyncio.run(calc.execute(expression="2 + 2"))
assert "4" in str(result)
self.log_verbose(" Calculator execution works")
def test_parameter_validation():
from effgen.tools.base_tool import ParameterSpec, ParameterType
param = ParameterSpec(
name="test_param",
type=ParameterType.INTEGER,
description="Test parameter",
required=True,
min_value=0,
max_value=100
)
# Test valid value
is_valid, error = param.validate(50)
assert is_valid
assert error is None
# Test invalid value
is_valid, error = param.validate(150)
assert not is_valid
assert error is not None
self.log_verbose(" Parameter validation working")
def test_tool_metadata_serialization():
from effgen.tools.base_tool import ToolCategory, ToolMetadata
metadata = ToolMetadata(
name="test_tool",
description="Test tool",
category=ToolCategory.COMPUTATION,
version="1.0.0"
)
# Test serialization
metadata_dict = metadata.to_dict()
assert metadata_dict["name"] == "test_tool"
assert metadata_dict["version"] == "1.0.0"
self.log_verbose(" Tool metadata serialization working")
def test_tool_result():
from effgen.tools.base_tool import ToolResult
result = ToolResult(
success=True,
output="Test output",
execution_time=0.5
)
assert result.success
assert result.output == "Test output"
self.log_verbose(" ToolResult working")
tests = [
("Calculator execution", test_calculator_execution),
("Parameter validation", test_parameter_validation),
("Tool metadata serialization", test_tool_metadata_serialization),
("Tool result structure", test_tool_result),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_async_operations(self):
"""Verify async operations work correctly"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Async Operations ==={Colors.ENDC}\n")
def test_async_tool_registry():
import asyncio
from effgen.tools import get_registry
from effgen.tools.builtin.calculator import Calculator
async def async_test():
registry = get_registry()
registry.register_tool(Calculator)
tool = await registry.get_tool("calculator")
assert tool is not None
return True
result = asyncio.run(async_test())
assert result
self.log_verbose(" Async tool registry operations working")
def test_async_tool_execution():
import asyncio
from effgen.tools.builtin.calculator import Calculator
async def async_test():
calc = Calculator()
result = await calc.execute(expression="10 * 5")
return result
result = asyncio.run(async_test())
assert "50" in str(result)
self.log_verbose(" Async tool execution working")
tests = [
("Async tool registry", test_async_tool_registry),
("Async tool execution", test_async_tool_execution),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_memory_integration(self):
"""Verify memory system integration"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Memory Integration ==={Colors.ENDC}\n")
def test_message_serialization():
from effgen.memory import Message, MessageRole
message = Message(
role=MessageRole.USER,
content="Test message",
metadata={"source": "test"}
)
# Test serialization
msg_dict = message.to_dict()
assert msg_dict["content"] == "Test message"
assert msg_dict["role"] == "user"
# Test deserialization
restored = Message.from_dict(msg_dict)
assert restored.content == "Test message"
self.log_verbose(" Message serialization working")
def test_memory_token_counting():
from effgen.memory import MessageRole, ShortTermMemory
memory = ShortTermMemory(max_tokens=1000)
memory.add_message(MessageRole.USER, "Hello")
memory.add_message(MessageRole.ASSISTANT, "Hi there")
token_count = memory.get_token_count()
assert token_count > 0
self.log_verbose(" Memory token counting working")
def test_long_term_memory_storage():
import tempfile
from effgen.memory import (
ImportanceLevel,
JSONStorageBackend,
LongTermMemory,
MemoryType,
)
# Create temp directory for storage
with tempfile.TemporaryDirectory() as tmpdir:
storage_path = os.path.join(tmpdir, "memory.json")
backend = JSONStorageBackend(filepath=storage_path)
ltm = LongTermMemory(backend=backend)
# Start a session
ltm.start_session(name="test_session")
# Add memory entry
ltm.add_memory(
content="Test memory",
memory_type=MemoryType.FACT,
importance=ImportanceLevel.HIGH
)
# Verify memory was added
# LongTermMemory stores in backend, just verify it doesn't crash
assert ltm is not None
self.log_verbose(" Long-term memory storage working")
def test_conversation_summary():
from effgen.memory.short_term import ConversationSummary
summary = ConversationSummary(
summary="Test summary",
message_count=10,
token_count=100
)
summary_dict = summary.to_dict()
assert summary_dict["message_count"] == 10
self.log_verbose(" Conversation summary working")
tests = [
("Message serialization", test_message_serialization),
("Memory token counting", test_memory_token_counting),
("Long-term memory storage", test_long_term_memory_storage),
("Conversation summary", test_conversation_summary),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_error_handling(self):
"""Verify error handling and edge cases"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Error Handling ==={Colors.ENDC}\n")
def test_invalid_tool_name():
from effgen.tools import get_registry
registry = get_registry()
# Should handle without raising
assert not registry.is_registered("nonexistent_tool")
self.log_verbose(" Invalid tool name handled")
def test_parameter_validation_errors():
from effgen.tools.base_tool import ParameterSpec, ParameterType
param = ParameterSpec(
name="test",
type=ParameterType.INTEGER,
description="Test",
required=True
)
# Test missing required parameter
is_valid, error = param.validate(None)
assert not is_valid
assert "required" in error.lower()
self.log_verbose(" Parameter validation errors working")
def test_code_validator_safety():
from effgen.execution.validators import PythonValidator
validator = PythonValidator()
# Test dangerous code
dangerous_code = "import os; os.system('rm -rf /')"
result = validator.validate(dangerous_code)
assert not result.is_safe
assert len(result.issues) > 0
self.log_verbose(" Code validator safety checks working")
def test_memory_overflow_handling():
from effgen.memory import MessageRole, ShortTermMemory
# Create memory with very small limit
memory = ShortTermMemory(max_messages=2)
# Add more messages than limit
memory.add_message(MessageRole.USER, "Message 1")
memory.add_message(MessageRole.USER, "Message 2")
memory.add_message(MessageRole.USER, "Message 3")
messages = memory.get_recent_messages()
assert len(messages) <= 2
self.log_verbose(" Memory overflow handling working")
tests = [
("Invalid tool name", test_invalid_tool_name),
("Parameter validation errors", test_parameter_validation_errors),
("Code validator safety", test_code_validator_safety),
("Memory overflow handling", test_memory_overflow_handling),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def verify_advanced_features(self):
"""Verify advanced framework features"""
self.log(f"\n{Colors.HEADER}{Colors.BOLD}=== Testing Advanced Features ==={Colors.ENDC}\n")
def test_gpu_allocation():
from effgen.gpu import GPUAllocator
allocator = GPUAllocator()
assert allocator is not None
self.log_verbose(" GPU allocator initialized")
def test_vector_memory():
try:
from effgen.memory import VectorMemoryStore
# Test with simple embedding provider
store = VectorMemoryStore()
assert store is not None
self.log_verbose(" Vector memory store created")
except Exception as e:
# Vector store may require optional dependencies
if "FAISS" in str(e) or "chroma" in str(e):
self.log_verbose(" Vector store requires optional dependencies (skipping)")
# Don't fail on optional dependency
return
else:
raise
def test_chain_manager():
from effgen.prompts import ChainManager
manager = ChainManager()
assert manager is not None
self.log_verbose(" Chain manager initialized")
def test_orchestrator():
from effgen.core.orchestrator import MultiAgentOrchestrator
# Orchestrator can be imported
assert MultiAgentOrchestrator is not None
self.log_verbose(" Orchestrator available")
def test_complexity_analyzer():
from effgen.core.complexity_analyzer import ComplexityAnalyzer
analyzer = ComplexityAnalyzer()
# Test complexity analysis
simple_task = "What is 2 + 2?"
result = analyzer.analyze(simple_task)
assert result.overall >= 0
self.log_verbose(" Complexity analyzer working")
def test_decomposition_engine():
from effgen.core.decomposition_engine import DecompositionEngine
engine = DecompositionEngine()
# Test task decomposition
complex_task = "Build a web application with authentication and database"
subtasks = engine.decompose(complex_task, strategy="parallel_sub_agents")
assert len(subtasks) > 0
self.log_verbose(" Decomposition engine working")
def test_execution_tracker():
from effgen.core.execution_tracker import ExecutionTracker
tracker = ExecutionTracker()
assert tracker is not None
self.log_verbose(" Execution tracker initialized")
def test_router():
from effgen.core.router import SubAgentRouter
router = SubAgentRouter()
assert router is not None
self.log_verbose(" Router initialized")
def test_sub_agent_manager():
from effgen.core.sub_agent_manager import SubAgentManager
manager = SubAgentManager()
assert manager is not None
self.log_verbose(" Sub-agent manager initialized")
tests = [
("GPU allocation system", test_gpu_allocation),
("Vector memory store", test_vector_memory),
("Prompt chain manager", test_chain_manager),
("Task orchestrator", test_orchestrator),
("Complexity analyzer", test_complexity_analyzer),
("Decomposition engine", test_decomposition_engine),
("Execution tracker", test_execution_tracker),
("Router", test_router),
("Sub-agent manager", test_sub_agent_manager),
]
for name, test_func in tests:
result = self.run_test(name, test_func)
self.add_result(result)
def run_all_verifications(self):
"""Run all verification tests"""
self.log(f"{Colors.HEADER}{Colors.BOLD}{'='*60}{Colors.ENDC}")
self.log(f"{Colors.HEADER}{Colors.BOLD}effGen Framework Verification{Colors.ENDC}")
self.log(f"{Colors.HEADER}{Colors.BOLD}Super Comprehensive Test Suite{Colors.ENDC}")
self.log(f"{Colors.HEADER}{Colors.BOLD}{'='*60}{Colors.ENDC}")
# Run all verification suites
self.verify_imports()
self.verify_core_classes()
self.verify_tool_system()
self.verify_configuration()
self.verify_model_interfaces()
self.verify_prompt_system()
self.verify_memory_system()
self.verify_execution_system()
self.verify_agent_integration()
self.verify_protocol_systems()
self.verify_tool_functionality()
self.verify_async_operations()
self.verify_memory_integration()
self.verify_error_handling()
self.verify_advanced_features()
self.verify_dependencies()
# Generate report
success = self.generate_report()
return 0 if success else 1
def main():
"""Main entry point"""
import argparse
parser = argparse.ArgumentParser(
description="Verify effGen framework installation and functionality"
)
parser.add_argument(
"--verbose", "-v",
action="store_true",
help="Enable verbose output"
)
parser.add_argument(
"--skip-optional",
action="store_true",
help="Skip optional dependency checks"
)
args = parser.parse_args()
verifier = FrameworkVerifier(
verbose=args.verbose,
skip_optional=args.skip_optional
)
exit_code = verifier.run_all_verifications()
sys.exit(exit_code)
if __name__ == "__main__":
main()
#!/usr/bin/env python3
"""
effGen Framework Verification Script
This validation script verifies that every component of the effGen framework
is installed, configured, and functional.
Usage:
python verify_framework.py [--verbose] [--skip-optional]
"""
import importlib
import os
import sys
import time
import traceback
from dataclasses import dataclass
from enum import Enum
# Color codes for terminal output
class Colors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
class TestStatus(Enum):
"""Status of individual tests"""
PASSED = "✓ PASSED"
FAILED = "✗ FAILED"
SKIPPED = "⊘ SKIPPED"
WARNING = "⚠ WARNING"
class TestResult:
"""Result of a single test"""
name: str
status: TestStatus
message: str
details: str | None = None
duration: float = 0.0
class FrameworkVerifier:
"""Main verification class for effGen framework"""
def __init__(self, verbose: bool = False, skip_optional: bool = False):
self.verbose = verbose
self.skip_optional = skip_optional
self.results: list[TestResult] = []
self.start_time = time.time()
def log(self, message: str, color: str = ""):
"""Print colored log message"""
if color:
print(f"{color}{message}{Colors.ENDC}")
else:
print(message)
def log_verbose(self, message: str):
"""Print message only in verbose mode"""
if self.verbose:
print(f" {Colors.OKCYAN}{message}{Colors.ENDC}")
def add_result(self, result: TestResult):
"""Add test result to results list"""
self.results.append(result)
# Print result
color = {
TestStatus.PASSED: Colors.OKGREEN,
TestStatus.FAILED: Colors.FAIL,
TestStatus.SKIPPED: Colors.WARNING,
TestStatus.WARNING: Colors.WARNING
}[result.status]
status_str = f"{color}{result.status.value}{Colors.ENDC}"
print(f"{status_str} {result.name} ({result.duration:.3f}s)")
if result.message and (self.verbose or result.status != TestStatus.PASSED):
print(f" → {result.message}")
if result.details and self.verbose:
print(f" Details: {result.details}")
def run_test(self, name: str, test_func, *args, **kwargs) -> TestResult:
"""Run a single test and capture result"""
start = time.time()
try:
self.log_verbose(f"Running: {name}")
test_func(*args, **kwargs)
duration = time.time() - start
Xet Storage Details
- Size:
- 47.2 kB
- Xet hash:
- fbfccad16d03a4ed7cd3853fbc8d8b79ee61b6ae61187f2b39cda4907e6aafb9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.