Spaces:
Build error
Build error
Download eval/runner.py from AIenthusSP/RAG-Documind-AI: direct link, hf CLI and curl.
- Browser
- Download file 7.55 kB
-
https://huggingface.co/spaces/AIenthusSP/RAG-Documind-AI/resolve/main/eval/runner.py
- Command line
-
hf download hf://spaces/AIenthusSP/RAG-Documind-AI/eval/runner.py
-
curl -L -o runner.py https://huggingface.co/spaces/AIenthusSP/RAG-Documind-AI/resolve/main/eval/runner.py
7.55 kB
| """ | |
| Promptfoo Evaluation Runner | |
| Python wrapper for running Promptfoo evaluations and processing results. | |
| """ | |
| import subprocess | |
| import json | |
| import logging | |
| from pathlib import Path | |
| from typing import Dict, Any, Optional | |
| from datetime import datetime | |
| logger = logging.getLogger(__name__) | |
| class PromptfooRunner: | |
| """ | |
| Wrapper for running Promptfoo evaluations. | |
| Provides Python interface for: | |
| - Running evaluations | |
| - Parsing results | |
| - Generating reports | |
| """ | |
| def __init__(self, config_path: str = "eval/promptfooconfig.yaml"): | |
| """ | |
| Initialize the runner. | |
| Args: | |
| config_path: Path to promptfoo config file | |
| """ | |
| self.config_path = Path(config_path) | |
| self.results_dir = Path("eval/results") | |
| self.results_dir.mkdir(parents=True, exist_ok=True) | |
| def run_eval( | |
| self, | |
| output_path: Optional[str] = None, | |
| no_cache: bool = False, | |
| verbose: bool = True, | |
| ) -> Dict[str, Any]: | |
| """ | |
| Run Promptfoo evaluation. | |
| Args: | |
| output_path: Custom output path for results | |
| no_cache: Disable caching | |
| verbose: Show detailed output | |
| Returns: | |
| Evaluation results dictionary | |
| """ | |
| if not output_path: | |
| timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") | |
| output_path = self.results_dir / f"eval_{timestamp}.json" | |
| cmd = [ | |
| "npx", "promptfoo", "eval", | |
| "-c", str(self.config_path), | |
| "-o", str(output_path), | |
| "--json", | |
| ] | |
| if no_cache: | |
| cmd.append("--no-cache") | |
| logger.info(f"Running Promptfoo evaluation: {' '.join(cmd)}") | |
| try: | |
| result = subprocess.run( | |
| cmd, | |
| capture_output=True, | |
| text=True, | |
| timeout=300, # 5 minute timeout | |
| ) | |
| if result.returncode != 0: | |
| logger.error(f"Promptfoo failed: {result.stderr}") | |
| return {"error": result.stderr} | |
| # Parse results | |
| with open(output_path, "r") as f: | |
| results = json.load(f) | |
| return results | |
| except subprocess.TimeoutExpired: | |
| logger.error("Promptfoo evaluation timed out") | |
| return {"error": "Evaluation timed out"} | |
| except Exception as e: | |
| logger.error(f"Failed to run Promptfoo: {e}") | |
| return {"error": str(e)} | |
| def parse_results(self, results: Dict[str, Any]) -> Dict[str, Any]: | |
| """ | |
| Parse and summarize evaluation results. | |
| Args: | |
| results: Raw Promptfoo results | |
| Returns: | |
| Summary dictionary | |
| """ | |
| if "error" in results: | |
| return results | |
| summary = { | |
| "total_tests": 0, | |
| "passed": 0, | |
| "failed": 0, | |
| "pass_rate": 0.0, | |
| "avg_latency_ms": 0, | |
| "total_cost": 0.0, | |
| "failures": [], | |
| } | |
| tests = results.get("results", []) | |
| summary["total_tests"] = len(tests) | |
| latencies = [] | |
| costs = [] | |
| for test in tests: | |
| if test.get("success"): | |
| summary["passed"] += 1 | |
| else: | |
| summary["failed"] += 1 | |
| summary["failures"].append({ | |
| "description": test.get("description", "Unknown"), | |
| "error": test.get("error", "Assertion failed"), | |
| }) | |
| if "latencyMs" in test: | |
| latencies.append(test["latencyMs"]) | |
| if "cost" in test: | |
| costs.append(test["cost"]) | |
| if summary["total_tests"] > 0: | |
| summary["pass_rate"] = summary["passed"] / summary["total_tests"] | |
| if latencies: | |
| summary["avg_latency_ms"] = sum(latencies) / len(latencies) | |
| if costs: | |
| summary["total_cost"] = sum(costs) | |
| return summary | |
| def generate_report( | |
| self, | |
| results: Dict[str, Any], | |
| output_format: str = "markdown", | |
| ) -> str: | |
| """ | |
| Generate human-readable report. | |
| Args: | |
| results: Evaluation results | |
| output_format: Output format (markdown, text) | |
| Returns: | |
| Formatted report string | |
| """ | |
| summary = self.parse_results(results) | |
| if output_format == "markdown": | |
| return self._generate_markdown_report(summary) | |
| else: | |
| return self._generate_text_report(summary) | |
| def _generate_markdown_report(self, summary: Dict[str, Any]) -> str: | |
| """Generate Markdown report.""" | |
| report = [ | |
| "# RAG Evaluation Report", | |
| "", | |
| f"**Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", | |
| "", | |
| "## Summary", | |
| "", | |
| "| Metric | Value |", | |
| "|--------|-------|", | |
| f"| Total Tests | {summary['total_tests']} |", | |
| f"| Passed | {summary['passed']} |", | |
| f"| Failed | {summary['failed']} |", | |
| f"| Pass Rate | {summary['pass_rate']:.1%} |", | |
| f"| Avg Latency | {summary['avg_latency_ms']:.0f}ms |", | |
| f"| Total Cost | ${summary['total_cost']:.4f} |", | |
| "", | |
| ] | |
| if summary["failures"]: | |
| report.extend([ | |
| "## Failures", | |
| "", | |
| ]) | |
| for failure in summary["failures"]: | |
| report.append(f"- **{failure['description']}**: {failure['error']}") | |
| report.append("") | |
| return "\n".join(report) | |
| def _generate_text_report(self, summary: Dict[str, Any]) -> str: | |
| """Generate plain text report.""" | |
| lines = [ | |
| "=== RAG Evaluation Report ===", | |
| "", | |
| f"Total Tests: {summary['total_tests']}", | |
| f"Passed: {summary['passed']}", | |
| f"Failed: {summary['failed']}", | |
| f"Pass Rate: {summary['pass_rate']:.1%}", | |
| f"Avg Latency: {summary['avg_latency_ms']:.0f}ms", | |
| f"Total Cost: ${summary['total_cost']:.4f}", | |
| ] | |
| if summary["failures"]: | |
| lines.extend(["", "Failures:"]) | |
| for failure in summary["failures"]: | |
| lines.append(f" - {failure['description']}: {failure['error']}") | |
| return "\n".join(lines) | |
| def run_evaluation( | |
| config_path: str = "eval/promptfooconfig.yaml", | |
| verbose: bool = True, | |
| ) -> Dict[str, Any]: | |
| """ | |
| Convenience function to run evaluation. | |
| Args: | |
| config_path: Path to config file | |
| verbose: Show detailed output | |
| Returns: | |
| Evaluation summary | |
| """ | |
| runner = PromptfooRunner(config_path) | |
| results = runner.run_eval(verbose=verbose) | |
| return runner.parse_results(results) | |
| if __name__ == "__main__": | |
| # Run evaluation when executed directly | |
| import sys | |
| logging.basicConfig(level=logging.INFO) | |
| runner = PromptfooRunner() | |
| results = runner.run_eval() | |
| if "error" in results: | |
| print(f"Evaluation failed: {results['error']}") | |
| sys.exit(1) | |
| report = runner.generate_report(results) | |
| print(report) | |