Spaces:

techprotrade
/

annator-command-center

Paused

annator-command-center / tests /scripts /check_pass_rate.py

Deploy ATOM FastAPI command center runtime (part 9)

81e3673 verified 3 months ago

10.8 kB

#!/usr/bin/env python
"""
Pass Rate Validation Script for Atom Test Suite
This script validates test suite pass rate against a minimum threshold (default 90%).
It parses pytest JSON output, categorizes failures, and returns appropriate exit codes.
Usage:
    python check_pass_rate.py --json-file pytest_report.json --minimum 90
    python check_pass_rate.py --help
Exit Codes:
    0: Pass rate meets or exceeds minimum threshold
    1: Pass rate below minimum threshold
    2: Error in execution
"""

import argparse
import json
import sys
from datetime import datetime
from pathlib import Path


def parse_pytest_json(json_file: str):
    """
    Parse pytest JSON report file and extract test results.
    Args:
        json_file: Path to pytest JSON report file
    Returns:
        Dictionary with passed, failed, error, skipped counts
    """
    try:
        with open(json_file, 'r') as f:
            data = json.load(f)
    except FileNotFoundError:
        print(f"ERROR: JSON report file not found: {json_file}")
        sys.exit(2)
    except json.JSONDecodeError as e:
        print(f"ERROR: Invalid JSON in report file: {e}")
        sys.exit(2)

    # Initialize counters
    results = {
        "passed": 0,
        "failed": 0,
        "error": 0,
        "skipped": 0,
        "total": 0
    }

    # Parse pytest-json-report format
    if "summary" in data:
        # pytest-json-report format
        summary = data["summary"]
        results["passed"] = summary.get("passed", 0)
        results["failed"] = summary.get("failed", 0)
        results["skipped"] = summary.get("skipped", 0)
        results["total"] = summary.get("total", 0)
        # Errors are counted as failed in this format
        results["error"] = summary.get("error", 0)
    elif "tests" in data:
        # Alternative format: iterate through test list
        for test in data["tests"]:
            outcome = test.get("outcome", "unknown")
            if outcome == "passed":
                results["passed"] += 1
            elif outcome == "failed":
                results["failed"] += 1
            elif outcome == "skipped":
                results["skipped"] += 1
            elif outcome == "error":
                results["error"] += 1
            results["total"] += 1
    else:
        print("ERROR: Unknown JSON report format")
        sys.exit(2)

    return results


def calculate_pass_rate(results):
    """
    Calculate pass rate percentage.
    Args:
        results: Dictionary with test counts
    Returns:
        Pass rate percentage (0-100)
    """
    total_run = results["passed"] + results["failed"]
    if total_run == 0:
        return 100.0  # No tests run, consider as passing
    return (results["passed"] / total_run) * 100


def categorize_failures(json_file: str):
    """
    Categorize test failures by type (assertion, error, timeout, skip).
    Args:
        json_file: Path to pytest JSON report file
    Returns:
        Dictionary mapping failure type to count
    """
    categories = {
        "assertion": 0,
        "error": 0,
        "timeout": 0,
        "skip": 0
    }

    try:
        with open(json_file, 'r') as f:
            data = json.load(f)
    except (FileNotFoundError, json.JSONDecodeError):
        return categories

    # Parse test failures
    tests = data.get("tests", [])
    for test in tests:
        outcome = test.get("outcome", "")

        if outcome == "failed":
            # Check if it's an assertion error or exception
            call = test.get("call", {})
            longrepr = call.get("longrepr", "")

            if "AssertionError" in longrepr or "assert" in longrepr:
                categories["assertion"] += 1
            elif "TimeoutError" in longrepr or "timeout" in longrepr.lower():
                categories["timeout"] += 1
            else:
                categories["error"] += 1
        elif outcome == "skipped":
            categories["skip"] += 1

    return categories


def print_summary(
    pass_rate,
    results,
    categories,
    minimum,
    verbose=False
):
    """
    Print formatted summary of test results.
    Args:
        pass_rate: Calculated pass rate percentage
        results: Test result counts
        categories: Failure type counts
        minimum: Minimum pass rate threshold
        verbose: Enable verbose output
    """
    print("\n" + "="*70)
    print("TEST PASS RATE VALIDATION")
    print("="*70)

    # Test results
    print(f"\nTest Results:")
    print(f"  Total Tests:  {results['total']}")
    print(f"  Passed:       {results['passed']}")

    if results["failed"] > 0:
        print(f"  Failed:       {results['failed']}")

    if results["error"] > 0:
        print(f"  Errors:       {results['error']}")

    if results["skipped"] > 0:
        print(f"  Skipped:      {results['skipped']}")

    # Pass rate
    print(f"\nPass Rate: {pass_rate:.2f}%")
    print(f"Threshold: {minimum:.2f}%")

    # Failure breakdown
    if sum(categories.values()) > 0 or verbose:
        print(f"\nFailure Categories:")
        print(f"  Assertion Errors: {categories['assertion']}")
        print(f"  Exceptions:       {categories['error']}")
        print(f"  Timeouts:         {categories['timeout']}")
        print(f"  Skipped:          {categories['skip']}")

    # Status
    print("\n" + "="*70)
    if pass_rate >= minimum:
        print("STATUS: PASS ✓")
        print("="*70 + "\n")
    else:
        print(f"STATUS: FAIL ✗ (Pass rate {pass_rate:.2f}% below {minimum:.2f}%)")
        print("="*70 + "\n")


def update_health_json(
    results,
    pass_rate,
    categories,
    phase="090",
    plan="02",
    duration_seconds=0
):
    """
    Update test_health.json with current test results.
    Args:
        results: Test result counts
        pass_rate: Calculated pass rate percentage
        categories: Failure type counts
        phase: Current phase number
        plan: Current plan number
        duration_seconds: Test execution duration
    """
    health_file = Path(__file__).parent.parent / "coverage_reports" / "metrics" / "test_health.json"

    # Load existing health data or create new structure
    if health_file.exists():
        try:
            with open(health_file, 'r') as f:
                health_data = json.load(f)
        except (json.JSONDecodeError, IOError):
            health_data = {}
    else:
        health_data = {}

    # Ensure structure exists
    if "pass_rate_history" not in health_data:
        health_data["pass_rate_history"] = []
    if "failure_categories" not in health_data:
        health_data["failure_categories"] = {
            "assertion": 0,
            "error": 0,
            "timeout": 0,
            "skip": 0
        }
    if "baseline" not in health_data:
        health_data["baseline"] = {}
    if "metadata" not in health_data:
        health_data["metadata"] = {}

    # Add current run to history
    current_entry = {
        "date": datetime.now().strftime("%Y-%m-%d"),
        "timestamp": datetime.now().isoformat(),
        "phase": phase,
        "plan": plan,
        "total_tests": results["total"],
        "passed": results["passed"],
        "failed": results["failed"],
        "skipped": results["skipped"],
        "pass_rate": round(pass_rate, 2),
        "duration_seconds": duration_seconds
    }
    health_data["pass_rate_history"].append(current_entry)

    # Update failure categories (cumulative)
    for key in health_data["failure_categories"]:
        health_data["failure_categories"][key] += categories.get(key, 0)

    # Set baseline if not exists
    if not health_data["baseline"]:
        health_data["baseline"] = {
            "date": current_entry["date"],
            "pass_rate": round(pass_rate, 2),
            "total_tests": results["total"]
        }

    # Update metadata
    health_data["metadata"]["format_version"] = 1
    health_data["metadata"]["minimum_pass_rate"] = 90.0
    health_data["metadata"]["last_updated"] = datetime.now().isoformat()

    # Write back to file
    health_file.parent.mkdir(parents=True, exist_ok=True)
    with open(health_file, 'w') as f:
        json.dump(health_data, f, indent=2)


def main():
    """Main entry point for pass rate validation."""
    parser = argparse.ArgumentParser(
        description="Validate test pass rate against minimum threshold",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python check_pass_rate.py --json-file pytest_report.json
  python check_pass_rate.py --json-file pytest_report.json --minimum 95
  python check_pass_rate.py --json-file pytest_report.json --verbose --update-health
Exit Codes:
  0: Pass rate meets or exceeds minimum threshold
  1: Pass rate below minimum threshold
  2: Error in execution
        """
    )

    parser.add_argument(
        "--json-file",
        type=str,
        default="pytest_report.json",
        help="Path to pytest JSON report file (default: pytest_report.json)"
    )

    parser.add_argument(
        "--minimum",
        type=float,
        default=90.0,
        help="Minimum pass rate threshold (default: 90.0)"
    )

    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Enable verbose output"
    )

    parser.add_argument(
        "--update-health",
        action="store_true",
        help="Update test_health.json with current results"
    )

    parser.add_argument(
        "--phase",
        type=str,
        default="090",
        help="Current phase number for health tracking (default: 090)"
    )

    parser.add_argument(
        "--plan",
        type=str,
        default="02",
        help="Current plan number for health tracking (default: 02)"
    )

    parser.add_argument(
        "--duration",
        type=int,
        default=0,
        help="Test execution duration in seconds (default: 0)"
    )

    args = parser.parse_args()

    # Parse test results
    results = parse_pytest_json(args.json_file)

    # Calculate pass rate
    pass_rate = calculate_pass_rate(results)

    # Categorize failures
    categories = categorize_failures(args.json_file)

    # Print summary
    print_summary(pass_rate, results, categories, args.minimum, args.verbose)

    # Update health JSON if requested
    if args.update_health:
        update_health_json(
            results,
            pass_rate,
            categories,
            phase=args.phase,
            plan=args.plan,
            duration_seconds=args.duration
        )
        if args.verbose:
            print(f"Updated test_health.json with current results\n")

    # Return exit code
    if pass_rate >= args.minimum:
        sys.exit(0)
    else:
        sys.exit(1)


if __name__ == "__main__":
    main()

	#!/usr/bin/env python
	"""
	Pass Rate Validation Script for Atom Test Suite

	This script validates test suite pass rate against a minimum threshold (default 90%).
	It parses pytest JSON output, categorizes failures, and returns appropriate exit codes.

	Usage:
	python check_pass_rate.py --json-file pytest_report.json --minimum 90
	python check_pass_rate.py --help

	Exit Codes:
	0: Pass rate meets or exceeds minimum threshold
	1: Pass rate below minimum threshold
	2: Error in execution
	"""

	import argparse
	import json
	import sys
	from datetime import datetime
	from pathlib import Path


	def parse_pytest_json(json_file: str):
	"""
	Parse pytest JSON report file and extract test results.

	Args:
	json_file: Path to pytest JSON report file

	Returns:
	Dictionary with passed, failed, error, skipped counts
	"""
	try:
	with open(json_file, 'r') as f:
	data = json.load(f)
	except FileNotFoundError:
	print(f"ERROR: JSON report file not found: {json_file}")
	sys.exit(2)
	except json.JSONDecodeError as e:
	print(f"ERROR: Invalid JSON in report file: {e}")
	sys.exit(2)

	# Initialize counters
	results = {
	"passed": 0,
	"failed": 0,
	"error": 0,
	"skipped": 0,
	"total": 0
	}

	# Parse pytest-json-report format
	if "summary" in data:
	# pytest-json-report format
	summary = data["summary"]
	results["passed"] = summary.get("passed", 0)
	results["failed"] = summary.get("failed", 0)
	results["skipped"] = summary.get("skipped", 0)
	results["total"] = summary.get("total", 0)
	# Errors are counted as failed in this format
	results["error"] = summary.get("error", 0)
	elif "tests" in data:
	# Alternative format: iterate through test list
	for test in data["tests"]:
	outcome = test.get("outcome", "unknown")
	if outcome == "passed":
	results["passed"] += 1
	elif outcome == "failed":
	results["failed"] += 1
	elif outcome == "skipped":
	results["skipped"] += 1
	elif outcome == "error":
	results["error"] += 1
	results["total"] += 1
	else:
	print("ERROR: Unknown JSON report format")
	sys.exit(2)

	return results


	def calculate_pass_rate(results):
	"""
	Calculate pass rate percentage.

	Args:
	results: Dictionary with test counts

	Returns:
	Pass rate percentage (0-100)
	"""
	total_run = results["passed"] + results["failed"]
	if total_run == 0:
	return 100.0 # No tests run, consider as passing
	return (results["passed"] / total_run) * 100


	def categorize_failures(json_file: str):
	"""
	Categorize test failures by type (assertion, error, timeout, skip).

	Args:
	json_file: Path to pytest JSON report file

	Returns:
	Dictionary mapping failure type to count
	"""
	categories = {
	"assertion": 0,
	"error": 0,
	"timeout": 0,
	"skip": 0
	}

	try:
	with open(json_file, 'r') as f:
	data = json.load(f)
	except (FileNotFoundError, json.JSONDecodeError):
	return categories

	# Parse test failures
	tests = data.get("tests", [])
	for test in tests:
	outcome = test.get("outcome", "")

	if outcome == "failed":
	# Check if it's an assertion error or exception
	call = test.get("call", {})
	longrepr = call.get("longrepr", "")

	if "AssertionError" in longrepr or "assert" in longrepr:
	categories["assertion"] += 1
	elif "TimeoutError" in longrepr or "timeout" in longrepr.lower():
	categories["timeout"] += 1
	else:
	categories["error"] += 1
	elif outcome == "skipped":
	categories["skip"] += 1

	return categories


	def print_summary(
	pass_rate,
	results,
	categories,
	minimum,
	verbose=False
	):
	"""
	Print formatted summary of test results.

	Args:
	pass_rate: Calculated pass rate percentage
	results: Test result counts
	categories: Failure type counts
	minimum: Minimum pass rate threshold
	verbose: Enable verbose output
	"""
	print("\n" + "="*70)
	print("TEST PASS RATE VALIDATION")
	print("="*70)

	# Test results
	print(f"\nTest Results:")
	print(f" Total Tests: {results['total']}")
	print(f" Passed: {results['passed']}")

	if results["failed"] > 0:
	print(f" Failed: {results['failed']}")

	if results["error"] > 0:
	print(f" Errors: {results['error']}")

	if results["skipped"] > 0:
	print(f" Skipped: {results['skipped']}")

	# Pass rate
	print(f"\nPass Rate: {pass_rate:.2f}%")
	print(f"Threshold: {minimum:.2f}%")

	# Failure breakdown
	if sum(categories.values()) > 0 or verbose:
	print(f"\nFailure Categories:")
	print(f" Assertion Errors: {categories['assertion']}")
	print(f" Exceptions: {categories['error']}")
	print(f" Timeouts: {categories['timeout']}")
	print(f" Skipped: {categories['skip']}")

	# Status
	print("\n" + "="*70)
	if pass_rate >= minimum:
	print("STATUS: PASS ✓")
	print("="*70 + "\n")
	else:
	print(f"STATUS: FAIL ✗ (Pass rate {pass_rate:.2f}% below {minimum:.2f}%)")
	print("="*70 + "\n")


	def update_health_json(
	results,
	pass_rate,
	categories,
	phase="090",
	plan="02",
	duration_seconds=0
	):
	"""
	Update test_health.json with current test results.

	Args:
	results: Test result counts
	pass_rate: Calculated pass rate percentage
	categories: Failure type counts
	phase: Current phase number
	plan: Current plan number
	duration_seconds: Test execution duration
	"""
	health_file = Path(__file__).parent.parent / "coverage_reports" / "metrics" / "test_health.json"

	# Load existing health data or create new structure
	if health_file.exists():
	try:
	with open(health_file, 'r') as f:
	health_data = json.load(f)
	except (json.JSONDecodeError, IOError):
	health_data = {}
	else:
	health_data = {}

	# Ensure structure exists
	if "pass_rate_history" not in health_data:
	health_data["pass_rate_history"] = []
	if "failure_categories" not in health_data:
	health_data["failure_categories"] = {
	"assertion": 0,
	"error": 0,
	"timeout": 0,
	"skip": 0
	}
	if "baseline" not in health_data:
	health_data["baseline"] = {}
	if "metadata" not in health_data:
	health_data["metadata"] = {}

	# Add current run to history
	current_entry = {
	"date": datetime.now().strftime("%Y-%m-%d"),
	"timestamp": datetime.now().isoformat(),
	"phase": phase,
	"plan": plan,
	"total_tests": results["total"],
	"passed": results["passed"],
	"failed": results["failed"],
	"skipped": results["skipped"],
	"pass_rate": round(pass_rate, 2),
	"duration_seconds": duration_seconds
	}
	health_data["pass_rate_history"].append(current_entry)

	# Update failure categories (cumulative)
	for key in health_data["failure_categories"]:
	health_data["failure_categories"][key] += categories.get(key, 0)

	# Set baseline if not exists
	if not health_data["baseline"]:
	health_data["baseline"] = {
	"date": current_entry["date"],
	"pass_rate": round(pass_rate, 2),
	"total_tests": results["total"]
	}

	# Update metadata
	health_data["metadata"]["format_version"] = 1
	health_data["metadata"]["minimum_pass_rate"] = 90.0
	health_data["metadata"]["last_updated"] = datetime.now().isoformat()

	# Write back to file
	health_file.parent.mkdir(parents=True, exist_ok=True)
	with open(health_file, 'w') as f:
	json.dump(health_data, f, indent=2)


	def main():
	"""Main entry point for pass rate validation."""
	parser = argparse.ArgumentParser(
	description="Validate test pass rate against minimum threshold",
	formatter_class=argparse.RawDescriptionHelpFormatter,
	epilog="""
	Examples:
	python check_pass_rate.py --json-file pytest_report.json
	python check_pass_rate.py --json-file pytest_report.json --minimum 95
	python check_pass_rate.py --json-file pytest_report.json --verbose --update-health

	Exit Codes:
	0: Pass rate meets or exceeds minimum threshold
	1: Pass rate below minimum threshold
	2: Error in execution
	"""
	)

	parser.add_argument(
	"--json-file",
	type=str,
	default="pytest_report.json",
	help="Path to pytest JSON report file (default: pytest_report.json)"
	)

	parser.add_argument(
	"--minimum",
	type=float,
	default=90.0,
	help="Minimum pass rate threshold (default: 90.0)"
	)

	parser.add_argument(
	"--verbose",
	action="store_true",
	help="Enable verbose output"
	)

	parser.add_argument(
	"--update-health",
	action="store_true",
	help="Update test_health.json with current results"
	)

	parser.add_argument(
	"--phase",
	type=str,
	default="090",
	help="Current phase number for health tracking (default: 090)"
	)

	parser.add_argument(
	"--plan",
	type=str,
	default="02",
	help="Current plan number for health tracking (default: 02)"
	)

	parser.add_argument(
	"--duration",
	type=int,
	default=0,
	help="Test execution duration in seconds (default: 0)"
	)

	args = parser.parse_args()

	# Parse test results
	results = parse_pytest_json(args.json_file)

	# Calculate pass rate
	pass_rate = calculate_pass_rate(results)

	# Categorize failures
	categories = categorize_failures(args.json_file)

	# Print summary
	print_summary(pass_rate, results, categories, args.minimum, args.verbose)

	# Update health JSON if requested
	if args.update_health:
	update_health_json(
	results,
	pass_rate,
	categories,
	phase=args.phase,
	plan=args.plan,
	duration_seconds=args.duration
	)
	if args.verbose:
	print(f"Updated test_health.json with current results\n")

	# Return exit code
	if pass_rate >= args.minimum:
	sys.exit(0)
	else:
	sys.exit(1)


	if __name__ == "__main__":
	main()