ComputerAssignments/CA18_memory_systems/scripts/comprehensive_benchmark.py

#!/usr/bin/env python3
"""
سیستم بنچمارک جامع برای سیستم‌های RAG و حافظه
این فایل شامل تست‌های جامع عملکرد و کیفیت سیستم‌ها است
"""

import os
import json
import asyncio
import logging
import time
import numpy as np
import pandas as pd
from typing import Dict, List, Tuple, Any, Optional, Union
from dataclasses import dataclass, field
from datetime import datetime, timedelta
import pickle
from pathlib import Path
import uuid
import psutil
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed

# LangChain imports
from langchain_google_genai import ChatGoogleGenerativeAI, GoogleGenerativeAIEmbeddings
from langchain.schema import Document, HumanMessage, AIMessage, SystemMessage
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.chains import RetrievalQA, ConversationalRetrievalChain
from langchain.memory import (
    ConversationBufferMemory,
    ConversationSummaryMemory,
    ConversationBufferWindowMemory,
    ConversationSummaryBufferMemory,
    ConversationTokenBufferMemory,
)
from langchain.prompts import PromptTemplate, ChatPromptTemplate

# Vector stores
from langchain_chroma import Chroma
from langchain_community.vectorstores import FAISS

# LangGraph imports
from langgraph.graph import StateGraph, END
from langgraph.checkpoint.memory import MemorySaver

# Visualization
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots

# Performance monitoring
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# تنظیمات لاگ
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)


@dataclass
class BenchmarkConfig:
    """تنظیمات سیستم بنچمارک"""

    # Gemini API settings
    gemini_api_key: str = ""
    gemini_model: str = "gemini-1.5-pro"
    temperature: float = 0.7
    max_tokens: int = 2048

    # Benchmark settings
    test_iterations: int = 5
    concurrent_users: int = 10
    test_duration: int = 60  # seconds

    # Performance thresholds
    max_response_time: float = 5.0
    min_accuracy: float = 0.8
    max_memory_usage: int = 1024 * 1024 * 1024  # 1GB

    # Test data settings
    test_dataset_size: int = 100
    query_variations: int = 10

    # Output settings
    results_directory: str = "benchmark_results"
    enable_detailed_logging: bool = True


@dataclass
class BenchmarkResult:
    """نتیجه بنچمارک"""

    # Test info
    test_name: str = ""
    test_type: str = ""
    timestamp: datetime = field(default_factory=datetime.now)

    # Performance metrics
    response_time: float = 0.0
    throughput: float = 0.0
    latency_p95: float = 0.0
    latency_p99: float = 0.0

    # Quality metrics
    accuracy: float = 0.0
    precision: float = 0.0
    recall: float = 0.0
    f1_score: float = 0.0

    # System metrics
    memory_usage: int = 0
    cpu_usage: float = 0.0
    error_rate: float = 0.0

    # Resource metrics
    total_requests: int = 0
    successful_requests: int = 0
    failed_requests: int = 0

    # Additional metrics
    context_relevance: float = 0.0
    response_coherence: float = 0.0
    factual_accuracy: float = 0.0

    metadata: Dict[str, Any] = field(default_factory=dict)


class TestDataGenerator:
    """تولیدکننده داده‌های آزمایشی"""

    def __init__(self, config: BenchmarkConfig):
        self.config = config

    def generate_knowledge_base(self) -> List[Document]:
        """تولید پایگاه دانش آزمایشی"""

        knowledge_topics = [
            {
                "topic": "هوش مصنوعی",
                "content": "هوش مصنوعی (AI) شاخه‌ای از علوم کامپیوتر است که به ایجاد سیستم‌هایی می‌پردازد که قادر به انجام وظایفی هستند که معمولاً نیاز به هوش انسانی دارند. این فناوری شامل یادگیری ماشین، پردازش زبان طبیعی، بینایی کامپیوتر و رباتیک است.",
            },
            {
                "topic": "یادگیری ماشین",
                "content": "یادگیری ماشین (Machine Learning) زیرشاخه‌ای از هوش مصنوعی است که به کامپیوترها امکان یادگیری از داده‌ها بدون برنامه‌نویسی صریح را می‌دهد. الگوریتم‌های یادگیری ماشین می‌توانند الگوها را در داده‌ها شناسایی کرده و پیش‌بینی‌هایی انجام دهند.",
            },
            {
                "topic": "پردازش زبان طبیعی",
                "content": "پردازش زبان طبیعی (NLP) شاخه‌ای از هوش مصنوعی است که به کامپیوترها امکان درک و تولید زبان انسانی را می‌دهد. این فناوری در ترجمه ماشینی، خلاصه‌سازی متن، تحلیل احساسات و چت‌بات‌ها استفاده می‌شود.",
            },
            {
                "topic": "بینایی کامپیوتر",
                "content": "بینایی کامپیوتر (Computer Vision) شاخه‌ای از هوش مصنوعی است که به کامپیوترها امکان درک و تفسیر اطلاعات بصری را می‌دهد. این فناوری در تشخیص اشیاء، تشخیص چهره، پزشکی تشخیصی و خودروهای خودران استفاده می‌شود.",
            },
            {
                "topic": "رباتیک",
                "content": "رباتیک (Robotics) شاخه‌ای از مهندسی است که با طراحی، ساخت و برنامه‌نویسی ربات‌ها سروکار دارد. ربات‌ها می‌توانند در محیط‌های مختلف کار کنند و وظایف پیچیده‌ای را انجام دهند.",
            },
            {
                "topic": "شبکه‌های عصبی",
                "content": "شبکه‌های عصبی مصنوعی (Artificial Neural Networks) سیستم‌هایی هستند که از ساختار مغز انسان الهام گرفته‌اند. این شبکه‌ها از نورون‌های مصنوعی تشکیل شده‌اند که با هم کار می‌کنند تا الگوها را شناسایی کنند.",
            },
            {
                "topic": "یادگیری عمیق",
                "content": "یادگیری عمیق (Deep Learning) زیرشاخه‌ای از یادگیری ماشین است که از شبکه‌های عصبی با چندین لایه استفاده می‌کند. این روش در تشخیص تصویر، پردازش زبان طبیعی و بازی‌های کامپیوتری موفقیت‌های بزرگی داشته است.",
            },
            {
                "topic": "داده‌کاوی",
                "content": "داده‌کاوی (Data Mining) فرآیند کشف الگوها و روابط در مجموعه‌های بزرگ داده است. این فرآیند شامل استفاده از تکنیک‌های آماری، یادگیری ماشین و هوش مصنوعی برای استخراج دانش از داده‌ها است.",
            },
            {
                "topic": "کلان داده",
                "content": "کلان داده (Big Data) به مجموعه‌های داده‌ای اشاره دارد که بسیار بزرگ و پیچیده هستند و با روش‌های سنتی پردازش داده قابل مدیریت نیستند. این داده‌ها معمولاً در سه بعد حجم، سرعت و تنوع تعریف می‌شوند.",
            },
            {
                "topic": "محاسبات ابری",
                "content": "محاسبات ابری (Cloud Computing) مدلی برای ارائه خدمات محاسباتی از طریق اینترنت است. این خدمات شامل سرورها، ذخیره‌سازی، پایگاه‌های داده، شبکه‌ها، نرم‌افزارها و هوش مصنوعی است.",
            },
        ]

        documents = []
        for topic_info in knowledge_topics:
            doc = Document(
                page_content=topic_info["content"],
                metadata={
                    "topic": topic_info["topic"],
                    "source": "generated",
                    "timestamp": datetime.now().isoformat(),
                },
            )
            documents.append(doc)

        logger.info(f"Generated {len(documents)} knowledge base documents")
        return documents

    def generate_test_queries(self) -> List[Dict[str, Any]]:
        """تولید پرسش‌های آزمایشی"""

        query_templates = [
            {
                "template": "{} چیست؟",
                "topics": [
                    "هوش مصنوعی",
                    "یادگیری ماشین",
                    "پردازش زبان طبیعی",
                    "بینایی کامپیوتر",
                    "رباتیک",
                ],
            },
            {
                "template": "{} چگونه کار می‌کند؟",
                "topics": ["یادگیری ماشین", "شبکه‌های عصبی", "یادگیری عمیق", "داده‌کاوی"],
            },
            {
                "template": "کاربردهای {} چیست؟",
                "topics": [
                    "هوش مصنوعی",
                    "پردازش زبان طبیعی",
                    "بینایی کامپیوتر",
                    "رباتیک",
                    "کلان داده",
                ],
            },
            {
                "template": "مزایای {} چیست؟",
                "topics": ["یادگیری ماشین", "محاسبات ابری", "کلان داده", "داده‌کاوی"],
            },
            {
                "template": "تفاوت بین {} و {} چیست؟",
                "topics": [
                    ("یادگیری ماشین", "یادگیری عمیق"),
                    ("هوش مصنوعی", "رباتیک"),
                    ("داده‌کاوی", "کلان داده"),
                    ("پردازش زبان طبیعی", "بینایی کامپیوتر"),
                ],
            },
        ]

        queries = []
        for template_info in query_templates:
            if isinstance(template_info["topics"][0], tuple):
                # برای پرسش‌های مقایسه‌ای
                for topic1, topic2 in template_info["topics"]:
                    query = template_info["template"].format(f"{topic1} و {topic2}")
                    queries.append(
                        {
                            "query": query,
                            "type": "comparison",
                            "topics": [topic1, topic2],
                            "expected_topics": [topic1, topic2],
                        }
                    )
            else:
                # برای پرسش‌های معمولی
                for topic in template_info["topics"]:
                    query = template_info["template"].format(topic)
                    queries.append(
                        {
                            "query": query,
                            "type": "definition",
                            "topics": [topic],
                            "expected_topics": [topic],
                        }
                    )

        logger.info(f"Generated {len(queries)} test queries")
        return queries


class PerformanceMonitor:
    """مانیتور عملکرد سیستم"""

    def __init__(self):
        self.start_time = None
        self.end_time = None
        self.metrics_history = []

    def start_monitoring(self):
        """شروع مانیتورینگ"""
        self.start_time = time.time()
        self.metrics_history = []

    def stop_monitoring(self):
        """توقف مانیتورینگ"""
        self.end_time = time.time()

    def record_metrics(self):
        """ثبت معیارهای فعلی"""
        if self.start_time is None:
            return

        current_time = time.time()
        elapsed_time = current_time - self.start_time

        metrics = {
            "timestamp": current_time,
            "elapsed_time": elapsed_time,
            "memory_usage": psutil.Process().memory_info().rss,
            "cpu_usage": psutil.cpu_percent(),
            "memory_percent": psutil.virtual_memory().percent,
        }

        self.metrics_history.append(metrics)

    def get_summary(self) -> Dict[str, Any]:
        """دریافت خلاصه معیارها"""
        if not self.metrics_history:
            return {}

        memory_usages = [m["memory_usage"] for m in self.metrics_history]
        cpu_usages = [m["cpu_usage"] for m in self.metrics_history]

        return {
            "total_duration": self.end_time - self.start_time if self.end_time else 0,
            "avg_memory_usage": np.mean(memory_usages),
            "max_memory_usage": np.max(memory_usages),
            "avg_cpu_usage": np.mean(cpu_usages),
            "max_cpu_usage": np.max(cpu_usages),
            "measurement_count": len(self.metrics_history),
        }


class ComprehensiveBenchmark:
    """سیستم بنچمارک جامع"""

    def __init__(self, config: BenchmarkConfig):
        self.config = config
        self.llm = ChatGoogleGenerativeAI(
            model=config.gemini_model,
            temperature=config.temperature,
            max_output_tokens=config.max_tokens,
            google_api_key=config.gemini_api_key,
        )

        self.embeddings = GoogleGenerativeAIEmbeddings(
            model="models/embedding-001", google_api_key=config.gemini_api_key
        )

        self.text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=1000, chunk_overlap=200
        )

        # Initialize components
        self.test_data_generator = TestDataGenerator(config)
        self.performance_monitor = PerformanceMonitor()

        # Test results
        self.benchmark_results = []

        # Ensure results directory exists
        os.makedirs(config.results_directory, exist_ok=True)

        logger.info("Comprehensive Benchmark initialized")

    def setup_test_environment(
        self,
    ) -> Tuple[Any, List[Document], List[Dict[str, Any]]]:
        """راه‌اندازی محیط آزمایش"""

        logger.info("Setting up test environment...")

        # Generate test data
        knowledge_documents = self.test_data_generator.generate_knowledge_base()
        test_queries = self.test_data_generator.generate_test_queries()

        # Create vector store
        vector_store = Chroma.from_documents(
            knowledge_documents,
            self.embeddings,
            persist_directory=f"{self.config.results_directory}/test_vector_store",
        )

        logger.info("Test environment setup completed")
        return vector_store, knowledge_documents, test_queries

    def benchmark_rag_system(
        self, vector_store: Any, test_queries: List[Dict[str, Any]]
    ) -> List[BenchmarkResult]:
        """بنچمارک سیستم RAG"""

        logger.info("Starting RAG system benchmark...")

        # Create RAG chain
        retriever = vector_store.as_retriever(search_kwargs={"k": 5})
        qa_chain = RetrievalQA.from_chain_type(
            llm=self.llm,
            chain_type="stuff",
            retriever=retriever,
            return_source_documents=True,
        )

        results = []

        for iteration in range(self.config.test_iterations):
            logger.info(
                f"RAG benchmark iteration {iteration + 1}/{self.config.test_iterations}"
            )

            iteration_results = []

            for query_info in test_queries:
                query = query_info["query"]
                expected_topics = query_info["expected_topics"]

                # Start monitoring
                self.performance_monitor.start_monitoring()

                try:
                    # Execute query
                    start_time = time.time()
                    result = qa_chain.invoke({"query": query})
                    response_time = time.time() - start_time

                    # Stop monitoring
                    self.performance_monitor.stop_monitoring()

                    # Calculate quality metrics
                    accuracy = self._calculate_response_accuracy(
                        result["result"], expected_topics
                    )

                    # Calculate context relevance
                    context_relevance = self._calculate_context_relevance(
                        result["source_documents"], expected_topics
                    )

                    # Create benchmark result
                    benchmark_result = BenchmarkResult(
                        test_name=f"RAG_Iteration_{iteration + 1}",
                        test_type="rag_system",
                        response_time=response_time,
                        accuracy=accuracy,
                        context_relevance=context_relevance,
                        total_requests=1,
                        successful_requests=1,
                        failed_requests=0,
                        error_rate=0.0,
                        metadata={
                            "query": query,
                            "expected_topics": expected_topics,
                            "response_length": len(result["result"]),
                            "sources_count": len(result["source_documents"]),
                        },
                    )

                    iteration_results.append(benchmark_result)

                except Exception as e:
                    logger.error(f"RAG query failed: {e}")

                    benchmark_result = BenchmarkResult(
                        test_name=f"RAG_Iteration_{iteration + 1}",
                        test_type="rag_system",
                        response_time=0.0,
                        accuracy=0.0,
                        context_relevance=0.0,
                        total_requests=1,
                        successful_requests=0,
                        failed_requests=1,
                        error_rate=1.0,
                        metadata={"error": str(e)},
                    )

                    iteration_results.append(benchmark_result)

            results.extend(iteration_results)

        logger.info(f"RAG benchmark completed with {len(results)} results")
        return results

    def benchmark_memory_systems(
        self, test_queries: List[Dict[str, Any]]
    ) -> List[BenchmarkResult]:
        """بنچمارک سیستم‌های حافظه"""

        logger.info("Starting memory systems benchmark...")

        # Create different memory systems
        memory_systems = {
            "buffer": ConversationBufferMemory(
                memory_key="chat_history", return_messages=True
            ),
            "summary": ConversationSummaryMemory(
                llm=self.llm, memory_key="chat_history", return_messages=True
            ),
            "window": ConversationBufferWindowMemory(
                k=5, memory_key="chat_history", return_messages=True
            ),
            "token_buffer": ConversationTokenBufferMemory(
                llm=self.llm,
                max_token_limit=1000,
                memory_key="chat_history",
                return_messages=True,
            ),
        }

        results = []

        for memory_type, memory in memory_systems.items():
            logger.info(f"Benchmarking {memory_type} memory system...")

            # Create conversation chain
            from langchain.chains import ConversationChain

            chain = ConversationChain(llm=self.llm, memory=memory, verbose=False)

            for iteration in range(self.config.test_iterations):
                # Clear memory for each iteration
                memory.clear()

                iteration_results = []

                for query_info in test_queries[:5]:  # Test with first 5 queries
                    query = query_info["query"]

                    try:
                        # Start monitoring
                        self.performance_monitor.start_monitoring()

                        # Execute query
                        start_time = time.time()
                        response = chain.predict(input=query)
                        response_time = time.time() - start_time

                        # Stop monitoring
                        self.performance_monitor.stop_monitoring()

                        # Calculate coherence score
                        coherence_score = self._calculate_response_coherence(response)

                        # Create benchmark result
                        benchmark_result = BenchmarkResult(
                            test_name=f"Memory_{memory_type}_Iteration_{iteration + 1}",
                            test_type=f"memory_{memory_type}",
                            response_time=response_time,
                            response_coherence=coherence_score,
                            total_requests=1,
                            successful_requests=1,
                            failed_requests=0,
                            error_rate=0.0,
                            metadata={
                                "query": query,
                                "response_length": len(response),
                                "memory_type": memory_type,
                            },
                        )

                        iteration_results.append(benchmark_result)

                    except Exception as e:
                        logger.error(f"Memory query failed: {e}")

                        benchmark_result = BenchmarkResult(
                            test_name=f"Memory_{memory_type}_Iteration_{iteration + 1}",
                            test_type=f"memory_{memory_type}",
                            response_time=0.0,
                            response_coherence=0.0,
                            total_requests=1,
                            successful_requests=0,
                            failed_requests=1,
                            error_rate=1.0,
                            metadata={"error": str(e)},
                        )

                        iteration_results.append(benchmark_result)

                results.extend(iteration_results)

        logger.info(f"Memory systems benchmark completed with {len(results)} results")
        return results

    def benchmark_concurrent_performance(
        self, vector_store: Any, test_queries: List[Dict[str, Any]]
    ) -> List[BenchmarkResult]:
        """بنچمارک عملکرد همزمان"""

        logger.info("Starting concurrent performance benchmark...")

        # Create RAG chain
        retriever = vector_store.as_retriever(search_kwargs={"k": 3})
        qa_chain = RetrievalQA.from_chain_type(
            llm=self.llm,
            chain_type="stuff",
            retriever=retriever,
            return_source_documents=True,
        )

        def execute_query(query_info: Dict[str, Any]) -> BenchmarkResult:
            """اجرای یک پرسش"""
            query = query_info["query"]

            try:
                start_time = time.time()
                result = qa_chain.invoke({"query": query})
                response_time = time.time() - start_time

                return BenchmarkResult(
                    test_name="Concurrent_Performance",
                    test_type="concurrent_rag",
                    response_time=response_time,
                    total_requests=1,
                    successful_requests=1,
                    failed_requests=0,
                    error_rate=0.0,
                    metadata={"query": query, "response_length": len(result["result"])},
                )

            except Exception as e:
                return BenchmarkResult(
                    test_name="Concurrent_Performance",
                    test_type="concurrent_rag",
                    response_time=0.0,
                    total_requests=1,
                    successful_requests=0,
                    failed_requests=1,
                    error_rate=1.0,
                    metadata={"error": str(e)},
                )

        results = []

        # Test with different concurrency levels
        concurrency_levels = [1, 3, 5, 10]

        for concurrency in concurrency_levels:
            logger.info(f"Testing concurrency level: {concurrency}")

            # Select queries for this test
            selected_queries = test_queries[: concurrency * 2]  # 2 queries per thread

            # Start monitoring
            self.performance_monitor.start_monitoring()

            start_time = time.time()

            # Execute queries concurrently
            with ThreadPoolExecutor(max_workers=concurrency) as executor:
                future_to_query = {
                    executor.submit(execute_query, query_info): query_info
                    for query_info in selected_queries
                }

                concurrent_results = []
                for future in as_completed(future_to_query):
                    result = future.result()
                    concurrent_results.append(result)

            total_time = time.time() - start_time

            # Stop monitoring
            self.performance_monitor.stop_monitoring()

            # Calculate throughput
            successful_requests = sum(
                1 for r in concurrent_results if r.successful_requests > 0
            )
            throughput = successful_requests / total_time if total_time > 0 else 0

            # Calculate latency percentiles
            response_times = [
                r.response_time for r in concurrent_results if r.response_time > 0
            ]
            if response_times:
                latency_p95 = np.percentile(response_times, 95)
                latency_p99 = np.percentile(response_times, 99)
            else:
                latency_p95 = 0
                latency_p99 = 0

            # Create summary result
            summary_result = BenchmarkResult(
                test_name=f"Concurrent_Performance_{concurrency}",
                test_type="concurrent_performance",
                response_time=total_time,
                throughput=throughput,
                latency_p95=latency_p95,
                latency_p99=latency_p99,
                total_requests=len(selected_queries),
                successful_requests=successful_requests,
                failed_requests=len(selected_queries) - successful_requests,
                error_rate=(len(selected_queries) - successful_requests)
                / len(selected_queries),
                metadata={
                    "concurrency_level": concurrency,
                    "queries_per_thread": 2,
                    "total_queries": len(selected_queries),
                },
            )

            results.append(summary_result)
            results.extend(concurrent_results)

        logger.info(
            f"Concurrent performance benchmark completed with {len(results)} results"
        )
        return results

    def _calculate_response_accuracy(
        self, response: str, expected_topics: List[str]
    ) -> float:
        """محاسبه دقت پاسخ"""

        if not response or not expected_topics:
            return 0.0

        response_lower = response.lower()
        topic_matches = 0

        for topic in expected_topics:
            if topic.lower() in response_lower:
                topic_matches += 1

        return topic_matches / len(expected_topics)

    def _calculate_context_relevance(
        self, source_docs: List[Document], expected_topics: List[str]
    ) -> float:
        """محاسبه ارتباط زمینه"""

        if not source_docs or not expected_topics:
            return 0.0

        relevant_docs = 0

        for doc in source_docs:
            doc_content = doc.page_content.lower()
            for topic in expected_topics:
                if topic.lower() in doc_content:
                    relevant_docs += 1
                    break

        return relevant_docs / len(source_docs)

    def _calculate_response_coherence(self, response: str) -> float:
        """محاسبه انسجام پاسخ"""

        if not response:
            return 0.0

        # Simple coherence check based on response length and structure
        words = response.split()

        if len(words) < 5:
            return 0.3
        elif len(words) < 20:
            return 0.6
        elif len(words) < 50:
            return 0.8
        else:
            return 0.9

    def run_comprehensive_benchmark(self) -> Dict[str, Any]:
        """اجرای بنچمارک جامع"""

        logger.info("Starting comprehensive benchmark...")

        # Setup test environment
        vector_store, knowledge_docs, test_queries = self.setup_test_environment()

        # Run different benchmark tests
        all_results = []

        # 1. RAG System Benchmark
        rag_results = self.benchmark_rag_system(vector_store, test_queries)
        all_results.extend(rag_results)

        # 2. Memory Systems Benchmark
        memory_results = self.benchmark_memory_systems(test_queries)
        all_results.extend(memory_results)

        # 3. Concurrent Performance Benchmark
        concurrent_results = self.benchmark_concurrent_performance(
            vector_store, test_queries
        )
        all_results.extend(concurrent_results)

        # Generate comprehensive report
        report = self._generate_benchmark_report(all_results)

        # Save results
        self._save_benchmark_results(all_results, report)

        logger.info("Comprehensive benchmark completed")
        return report

    def _generate_benchmark_report(
        self, results: List[BenchmarkResult]
    ) -> Dict[str, Any]:
        """تولید گزارش بنچمارک"""

        if not results:
            return {"error": "No benchmark results available"}

        # Convert results to DataFrame for analysis
        df_data = []
        for result in results:
            df_data.append(
                {
                    "test_name": result.test_name,
                    "test_type": result.test_type,
                    "response_time": result.response_time,
                    "accuracy": result.accuracy,
                    "context_relevance": result.context_relevance,
                    "response_coherence": result.response_coherence,
                    "throughput": result.throughput,
                    "error_rate": result.error_rate,
                    "total_requests": result.total_requests,
                    "successful_requests": result.successful_requests,
                    "timestamp": result.timestamp,
                }
            )

        df = pd.DataFrame(df_data)

        # Generate summary statistics
        report = {
            "benchmark_info": {
                "total_tests": len(results),
                "test_types": df["test_type"].unique().tolist(),
                "benchmark_date": datetime.now().isoformat(),
                "config": {
                    "iterations": self.config.test_iterations,
                    "concurrent_users": self.config.concurrent_users,
                    "test_duration": self.config.test_duration,
                },
            },
            "performance_summary": {},
            "quality_summary": {},
            "system_summary": {},
            "recommendations": [],
        }

        # Performance summary
        for test_type in df["test_type"].unique():
            type_data = df[df["test_type"] == test_type]

            report["performance_summary"][test_type] = {
                "avg_response_time": type_data["response_time"].mean(),
                "median_response_time": type_data["response_time"].median(),
                "p95_response_time": type_data["response_time"].quantile(0.95),
                "p99_response_time": type_data["response_time"].quantile(0.99),
                "avg_throughput": type_data["throughput"].mean(),
                "test_count": len(type_data),
            }

        # Quality summary
        quality_metrics = ["accuracy", "context_relevance", "response_coherence"]
        for metric in quality_metrics:
            if metric in df.columns:
                report["quality_summary"][metric] = {
                    "average": df[metric].mean(),
                    "median": df[metric].median(),
                    "std": df[metric].std(),
                    "min": df[metric].min(),
                    "max": df[metric].max(),
                }

        # System summary
        report["system_summary"] = {
            "total_requests": df["total_requests"].sum(),
            "successful_requests": df["successful_requests"].sum(),
            "failed_requests": df["total_requests"].sum()
            - df["successful_requests"].sum(),
            "overall_error_rate": (
                (df["total_requests"].sum() - df["successful_requests"].sum())
                / df["total_requests"].sum()
                if df["total_requests"].sum() > 0
                else 0
            ),
        }

        # Generate recommendations
        recommendations = []

        # Performance recommendations
        if report["performance_summary"]:
            best_performance = min(
                report["performance_summary"].items(),
                key=lambda x: x[1]["avg_response_time"],
            )
            recommendations.append(
                f"بهترین عملکرد: {best_performance[0]} با میانگین زمان پاسخ {best_performance[1]['avg_response_time']:.3f}s"
            )

        # Quality recommendations
        if report["quality_summary"]:
            if "accuracy" in report["quality_summary"]:
                avg_accuracy = report["quality_summary"]["accuracy"]["average"]
                if avg_accuracy < 0.8:
                    recommendations.append(
                        "دقت پاسخ‌ها پایین است. بهبود کیفیت پایگاه دانش توصیه می‌شود"
                    )
                else:
                    recommendations.append("دقت پاسخ‌ها در سطح قابل قبولی است")

        # Error rate recommendations
        if report["system_summary"]["overall_error_rate"] > 0.1:
            recommendations.append("نرخ خطا بالا است. بررسی پایداری سیستم توصیه می‌شود")

        report["recommendations"] = recommendations

        return report

    def _save_benchmark_results(
        self, results: List[BenchmarkResult], report: Dict[str, Any]
    ):
        """ذخیره نتایج بنچمارک"""

        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")

        # Save detailed results
        results_file = os.path.join(
            self.config.results_directory, f"benchmark_results_{timestamp}.json"
        )

        serializable_results = []
        for result in results:
            serializable_results.append(
                {
                    "test_name": result.test_name,
                    "test_type": result.test_type,
                    "timestamp": result.timestamp.isoformat(),
                    "response_time": result.response_time,
                    "throughput": result.throughput,
                    "latency_p95": result.latency_p95,
                    "latency_p99": result.latency_p99,
                    "accuracy": result.accuracy,
                    "precision": result.precision,
                    "recall": result.recall,
                    "f1_score": result.f1_score,
                    "memory_usage": result.memory_usage,
                    "cpu_usage": result.cpu_usage,
                    "error_rate": result.error_rate,
                    "total_requests": result.total_requests,
                    "successful_requests": result.successful_requests,
                    "failed_requests": result.failed_requests,
                    "context_relevance": result.context_relevance,
                    "response_coherence": result.response_coherence,
                    "factual_accuracy": result.factual_accuracy,
                    "metadata": result.metadata,
                }
            )

        with open(results_file, "w", encoding="utf-8") as f:
            json.dump(serializable_results, f, indent=2, ensure_ascii=False)

        # Save report
        report_file = os.path.join(
            self.config.results_directory, f"benchmark_report_{timestamp}.json"
        )
        with open(report_file, "w", encoding="utf-8") as f:
            json.dump(report, f, indent=2, ensure_ascii=False)

        logger.info(f"Benchmark results saved to {results_file}")
        logger.info(f"Benchmark report saved to {report_file}")


def run_comprehensive_benchmark_demo():
    """اجرای دموی بنچمارک جامع"""

    print("🏁 Comprehensive Benchmark Demo")
    print("=" * 60)
    print("این دمو شامل بنچمارک جامع سیستم‌های RAG و حافظه است")
    print()

    # تنظیمات
    config = BenchmarkConfig(
        gemini_api_key=os.getenv("GOOGLE_API_KEY", ""),
        test_iterations=3,
        concurrent_users=5,
        test_duration=30,
        results_directory="benchmark_results",
    )

    if not config.gemini_api_key:
        print("⚠️ خطا: GOOGLE_API_KEY تنظیم نشده است")
        print("لطفاً کلید API خود را تنظیم کنید:")
        print("export GOOGLE_API_KEY='your_api_key_here'")
        return

    try:
        # ایجاد سیستم بنچمارک
        print("🔧 Initializing Comprehensive Benchmark...")
        benchmark = ComprehensiveBenchmark(config)

        # اجرای بنچمارک جامع
        print("🏃 Running comprehensive benchmark...")
        report = benchmark.run_comprehensive_benchmark()

        # نمایش نتایج
        print("\n📊 Benchmark Results Summary:")
        print("=" * 60)

        if "benchmark_info" in report:
            info = report["benchmark_info"]
            print(f"🔢 Total tests: {info['total_tests']}")
            print(f"📋 Test types: {', '.join(info['test_types'])}")
            print(f"📅 Benchmark date: {info['benchmark_date']}")

        if "performance_summary" in report:
            print(f"\n⚡ Performance Summary:")
            for test_type, perf_data in report["performance_summary"].items():
                print(f"  🔹 {test_type.upper()}:")
                print(
                    f"    ⏱️  Avg response time: {perf_data['avg_response_time']:.3f}s"
                )
                print(f"    📊 Avg throughput: {perf_data['avg_throughput']:.2f} req/s")
                print(f"    🔢 Test count: {perf_data['test_count']}")

        if "quality_summary" in report:
            print(f"\n🎯 Quality Summary:")
            for metric, quality_data in report["quality_summary"].items():
                print(f"  🔹 {metric.upper()}:")
                print(f"    📈 Average: {quality_data['average']:.3f}")
                print(f"    📊 Median: {quality_data['median']:.3f}")
                print(f"    📉 Min: {quality_data['min']:.3f}")
                print(f"    📈 Max: {quality_data['max']:.3f}")

        if "system_summary" in report:
            sys_summary = report["system_summary"]
            print(f"\n🖥️ System Summary:")
            print(f"  📊 Total requests: {sys_summary['total_requests']}")
            print(f"  ✅ Successful requests: {sys_summary['successful_requests']}")
            print(f"  ❌ Failed requests: {sys_summary['failed_requests']}")
            print(f"  📉 Overall error rate: {sys_summary['overall_error_rate']:.2%}")

        if "recommendations" in report:
            print(f"\n💡 Recommendations:")
            for i, rec in enumerate(report["recommendations"], 1):
                print(f"  {i}. {rec}")

        print(f"\n✅ Comprehensive Benchmark completed successfully!")
        print(f"📁 Results saved to: {config.results_directory}/")

        return report

    except Exception as e:
        print(f"❌ Benchmark failed: {e}")
        logger.error(f"Benchmark failed: {e}")
        return None


if __name__ == "__main__":
    # اجرای دمو
    run_comprehensive_benchmark_demo()