RAG-Documind-AI / eval /runner.py
AIenthusSP's picture
fix: fix linting issues
a6df84a
Raw History Blame Contribute Delete
7.55 kB
"""
Promptfoo Evaluation Runner
Python wrapper for running Promptfoo evaluations and processing results.
"""
import subprocess
import json
import logging
from pathlib import Path
from typing import Dict, Any, Optional
from datetime import datetime
logger = logging.getLogger(__name__)
class PromptfooRunner:
"""
Wrapper for running Promptfoo evaluations.
Provides Python interface for:
- Running evaluations
- Parsing results
- Generating reports
"""
def __init__(self, config_path: str = "eval/promptfooconfig.yaml"):
"""
Initialize the runner.
Args:
config_path: Path to promptfoo config file
"""
self.config_path = Path(config_path)
self.results_dir = Path("eval/results")
self.results_dir.mkdir(parents=True, exist_ok=True)
def run_eval(
self,
output_path: Optional[str] = None,
no_cache: bool = False,
verbose: bool = True,
) -> Dict[str, Any]:
"""
Run Promptfoo evaluation.
Args:
output_path: Custom output path for results
no_cache: Disable caching
verbose: Show detailed output
Returns:
Evaluation results dictionary
"""
if not output_path:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_path = self.results_dir / f"eval_{timestamp}.json"
cmd = [
"npx", "promptfoo", "eval",
"-c", str(self.config_path),
"-o", str(output_path),
"--json",
]
if no_cache:
cmd.append("--no-cache")
logger.info(f"Running Promptfoo evaluation: {' '.join(cmd)}")
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=300, # 5 minute timeout
)
if result.returncode != 0:
logger.error(f"Promptfoo failed: {result.stderr}")
return {"error": result.stderr}
# Parse results
with open(output_path, "r") as f:
results = json.load(f)
return results
except subprocess.TimeoutExpired:
logger.error("Promptfoo evaluation timed out")
return {"error": "Evaluation timed out"}
except Exception as e:
logger.error(f"Failed to run Promptfoo: {e}")
return {"error": str(e)}
def parse_results(self, results: Dict[str, Any]) -> Dict[str, Any]:
"""
Parse and summarize evaluation results.
Args:
results: Raw Promptfoo results
Returns:
Summary dictionary
"""
if "error" in results:
return results
summary = {
"total_tests": 0,
"passed": 0,
"failed": 0,
"pass_rate": 0.0,
"avg_latency_ms": 0,
"total_cost": 0.0,
"failures": [],
}
tests = results.get("results", [])
summary["total_tests"] = len(tests)
latencies = []
costs = []
for test in tests:
if test.get("success"):
summary["passed"] += 1
else:
summary["failed"] += 1
summary["failures"].append({
"description": test.get("description", "Unknown"),
"error": test.get("error", "Assertion failed"),
})
if "latencyMs" in test:
latencies.append(test["latencyMs"])
if "cost" in test:
costs.append(test["cost"])
if summary["total_tests"] > 0:
summary["pass_rate"] = summary["passed"] / summary["total_tests"]
if latencies:
summary["avg_latency_ms"] = sum(latencies) / len(latencies)
if costs:
summary["total_cost"] = sum(costs)
return summary
def generate_report(
self,
results: Dict[str, Any],
output_format: str = "markdown",
) -> str:
"""
Generate human-readable report.
Args:
results: Evaluation results
output_format: Output format (markdown, text)
Returns:
Formatted report string
"""
summary = self.parse_results(results)
if output_format == "markdown":
return self._generate_markdown_report(summary)
else:
return self._generate_text_report(summary)
def _generate_markdown_report(self, summary: Dict[str, Any]) -> str:
"""Generate Markdown report."""
report = [
"# RAG Evaluation Report",
"",
f"**Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
"",
"## Summary",
"",
"| Metric | Value |",
"|--------|-------|",
f"| Total Tests | {summary['total_tests']} |",
f"| Passed | {summary['passed']} |",
f"| Failed | {summary['failed']} |",
f"| Pass Rate | {summary['pass_rate']:.1%} |",
f"| Avg Latency | {summary['avg_latency_ms']:.0f}ms |",
f"| Total Cost | ${summary['total_cost']:.4f} |",
"",
]
if summary["failures"]:
report.extend([
"## Failures",
"",
])
for failure in summary["failures"]:
report.append(f"- **{failure['description']}**: {failure['error']}")
report.append("")
return "\n".join(report)
def _generate_text_report(self, summary: Dict[str, Any]) -> str:
"""Generate plain text report."""
lines = [
"=== RAG Evaluation Report ===",
"",
f"Total Tests: {summary['total_tests']}",
f"Passed: {summary['passed']}",
f"Failed: {summary['failed']}",
f"Pass Rate: {summary['pass_rate']:.1%}",
f"Avg Latency: {summary['avg_latency_ms']:.0f}ms",
f"Total Cost: ${summary['total_cost']:.4f}",
]
if summary["failures"]:
lines.extend(["", "Failures:"])
for failure in summary["failures"]:
lines.append(f" - {failure['description']}: {failure['error']}")
return "\n".join(lines)
def run_evaluation(
config_path: str = "eval/promptfooconfig.yaml",
verbose: bool = True,
) -> Dict[str, Any]:
"""
Convenience function to run evaluation.
Args:
config_path: Path to config file
verbose: Show detailed output
Returns:
Evaluation summary
"""
runner = PromptfooRunner(config_path)
results = runner.run_eval(verbose=verbose)
return runner.parse_results(results)
if __name__ == "__main__":
# Run evaluation when executed directly
import sys
logging.basicConfig(level=logging.INFO)
runner = PromptfooRunner()
results = runner.run_eval()
if "error" in results:
print(f"Evaluation failed: {results['error']}")
sys.exit(1)
report = runner.generate_report(results)
print(report)