tests/scripts/check_lighthouse_regression.py

#!/usr/bin/env python3
"""
Lighthouse Regression Detection Script

This script compares current Lighthouse results against a historical baseline
to detect performance regressions. It checks for:

1. Performance score regression (>20% degradation)
2. Core Web Vitals regression (>20% degradation):
   - First Contentful Paint (FCP)
   - Largest Contentful Paint (LCP)
   - Total Blocking Time (TBT)
   - Cumulative Layout Shift (CLS)

Usage:
    python check_lighthouse_regression.py \
        --current .lighthouseci/lhr-report.json \
        --baseline backend/tests/performance_regression/lighthouse_baseline.json \
        --threshold 0.2

Exit Codes:
    0: No regression detected
    1: Regression detected
    2: Error (missing files, invalid JSON, etc.)
"""

import argparse
import json
import sys
from pathlib import Path
from typing import Dict, Any, Optional, List, Tuple


# ============================================================================
# Lighthouse Metric Parsing
# ============================================================================

def parse_lighthouse_metrics(report_path: str) -> Dict[str, Any]:
    """Parse Lighthouse JSON report and extract key metrics.

    Args:
        report_path: Path to Lighthouse JSON report

    Returns:
        dict: Extracted metrics including scores and Core Web Vitals

    Raises:
        FileNotFoundError: If report file doesn't exist
        json.JSONDecodeError: If report is invalid JSON
        KeyError: If expected metrics are missing
    """
    with open(report_path, 'r') as f:
        report = json.load(f)

    categories = report.get('categories', {})
    audits = report.get('audits', {})

    # Extract performance score (0-100 scale)
    performance_score = categories.get('performance', {}).get('score', 0)
    if performance_score is not None:
        performance_score = performance_score * 100  # Convert to 0-100 scale

    # Extract Core Web Vitals
    metrics = {
        'performance_score': performance_score,
        'accessibility_score': categories.get('accessibility', {}).get('score', 0) * 100,
        'best_practices_score': categories.get('best-practices', {}).get('score', 0) * 100,
        'seo_score': categories.get('seo', {}).get('score', 0) * 100,
        'first_contentful_paint': audits.get('first-contentful-paint', {}).get('numericValue'),
        'largest_contentful_paint': audits.get('largest-contentful-paint', {}).get('numericValue'),
        'total_blocking_time': audits.get('total-blocking-time', {}).get('numericValue'),
        'cumulative_layout_shift': audits.get('cumulative-layout-shift', {}).get('numericValue'),
        'speed_index': audits.get('speed-index', {}).get('numericValue'),
    }

    return metrics


# ============================================================================
# Regression Detection
# ============================================================================

def check_regression(
    current_metrics: Dict[str, Any],
    baseline_metrics: Dict[str, Any],
    threshold: float = 0.2
) -> Tuple[bool, List[str]]:
    """Check for performance regressions by comparing current vs baseline.

    Args:
        current_metrics: Current Lighthouse metrics
        baseline_metrics: Baseline Lighthouse metrics
        threshold: Regression threshold (default 0.2 = 20%)

    Returns:
        tuple: (regression_detected, list of regression messages)
    """
    regressions = []

    # Check performance score (lower is bad)
    current_score = current_metrics.get('performance_score', 0)
    baseline_score = baseline_metrics.get('performance_score', 0)

    if baseline_score > 0 and current_score < baseline_score * (1 - threshold):
        regression_percent = ((baseline_score - current_score) / baseline_score) * 100
        regressions.append(
            f"REGRESSION: Performance score {current_score:.0f} < baseline {baseline_score:.0f} "
            f"({regression_percent:.1f}% degradation)"
        )

    # Check Core Web Vitals (higher is bad for timing metrics)
    vitals_to_check = [
        ('first_contentful_paint', 'FCP'),
        ('largest_contentful_paint', 'LCP'),
        ('total_blocking_time', 'TBT'),
        ('cumulative_layout_shift', 'CLS'),
    ]

    for metric_key, metric_name in vitals_to_check:
        current_value = current_metrics.get(metric_key)
        baseline_value = baseline_metrics.get(metric_key)

        # Skip if baseline value is missing or zero
        if baseline_value is None or baseline_value == 0:
            continue

        # Skip if current value is missing
        if current_value is None:
            continue

        # Check for regression (current > baseline * (1 + threshold))
        if current_value > baseline_value * (1 + threshold):
            regression_percent = ((current_value - baseline_value) / baseline_value) * 100
            unit = 'ms' if metric_key != 'cumulative_layout_shift' else ''

            regressions.append(
                f"REGRESSION: {metric_name} {current_value:.0f}{unit} > "
                f"baseline {baseline_value:.0f}{unit} "
                f"({regression_percent:.1f}% degradation)"
            )

    return len(regressions) > 0, regressions


# ============================================================================
# CLI Interface
# ============================================================================

def parse_args() -> argparse.Namespace:
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(
        description='Check Lighthouse results for performance regressions',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
    # Check for regressions with default 20%% threshold
    %(prog)s --current .lighthouseci/lhr-report.json \\
             --baseline backend/tests/performance_regression/lighthouse_baseline.json

    # Use custom threshold (15%%)
    %(prog)s --current .lighthouseci/lhr-report.json \\
             --baseline backend/tests/performance_regression/lighthouse_baseline.json \\
             --threshold 0.15

Exit Codes:
    0: No regression detected
    1: Regression detected
    2: Error (missing files, invalid JSON)
        """
    )

    parser.add_argument(
        '--current',
        required=True,
        help='Path to current Lighthouse JSON report'
    )

    parser.add_argument(
        '--baseline',
        required=True,
        help='Path to baseline Lighthouse JSON file'
    )

    parser.add_argument(
        '--threshold',
        type=float,
        default=0.2,
        help='Regression threshold (default: 0.2 = 20%%)'
    )

    return parser.parse_args()


def main() -> int:
    """Main entry point for CLI."""
    args = parse_args()

    # Validate threshold
    if args.threshold <= 0 or args.threshold >= 1:
        print(f"ERROR: Threshold must be between 0 and 1, got {args.threshold}", file=sys.stderr)
        return 2

    # Check if current report exists
    current_path = Path(args.current)
    if not current_path.exists():
        print(f"ERROR: Current report not found: {args.current}", file=sys.stderr)
        return 2

    # Check if baseline exists
    baseline_path = Path(args.baseline)
    if not baseline_path.exists():
        print(f"ERROR: Baseline file not found: {args.baseline}", file=sys.stderr)
        return 2

    # Parse current metrics
    try:
        current_metrics = parse_lighthouse_metrics(args.current)
    except FileNotFoundError:
        print(f"ERROR: Current report not found: {args.current}", file=sys.stderr)
        return 2
    except json.JSONDecodeError as e:
        print(f"ERROR: Invalid JSON in current report: {e}", file=sys.stderr)
        return 2
    except Exception as e:
        print(f"ERROR: Failed to parse current report: {e}", file=sys.stderr)
        return 2

    # Parse baseline metrics
    try:
        # Baseline file might be in different format (simplified JSON)
        with open(args.baseline, 'r') as f:
            baseline_data = json.load(f)

        # Handle both full Lighthouse report and simplified baseline format
        if 'metrics' in baseline_data:
            # Simplified baseline format
            baseline_metrics = baseline_data['metrics']
        elif 'categories' in baseline_data:
            # Full Lighthouse report format
            baseline_metrics = parse_lighthouse_metrics(args.baseline)
        else:
            # Assume it's already a metrics dict
            baseline_metrics = baseline_data

    except FileNotFoundError:
        print(f"ERROR: Baseline file not found: {args.baseline}", file=sys.stderr)
        return 2
    except json.JSONDecodeError as e:
        print(f"ERROR: Invalid JSON in baseline: {e}", file=sys.stderr)
        return 2
    except Exception as e:
        print(f"ERROR: Failed to parse baseline: {e}", file=sys.stderr)
        return 2

    # Print comparison header
    print("=" * 80)
    print("Lighthouse Regression Detection")
    print("=" * 80)
    print(f"Current:  {args.current}")
    print(f"Baseline: {args.baseline}")
    print(f"Threshold: {args.threshold * 100:.0f}%")
    print("=" * 80)

    # Print current metrics
    print("\n[Current Metrics]")
    print(f"  Performance Score:  {current_metrics.get('performance_score', 0):.0f}/100")
    print(f"  Accessibility Score: {current_metrics.get('accessibility_score', 0):.0f}/100")
    print(f"  Best Practices:      {current_metrics.get('best_practices_score', 0):.0f}/100")
    print(f"  SEO Score:           {current_metrics.get('seo_score', 0):.0f}/100")
    print(f"  FCP:                 {current_metrics.get('first_contentful_paint', 0) or 0:.0f}ms")
    print(f"  LCP:                 {current_metrics.get('largest_contentful_paint', 0) or 0:.0f}ms")
    print(f"  TBT:                 {current_metrics.get('total_blocking_time', 0) or 0:.0f}ms")
    print(f"  CLS:                 {current_metrics.get('cumulative_layout_shift', 0) or 0:.3f}")
    print(f"  Speed Index:         {current_metrics.get('speed_index', 0) or 0:.0f}ms")

    # Print baseline metrics
    print("\n[Baseline Metrics]")
    print(f"  Performance Score:  {baseline_metrics.get('performance_score', 0):.0f}/100")
    print(f"  Accessibility Score: {baseline_metrics.get('accessibility_score', 0):.0f}/100")
    print(f"  Best Practices:      {baseline_metrics.get('best_practices_score', 0):.0f}/100")
    print(f"  SEO Score:           {baseline_metrics.get('seo_score', 0):.0f}/100")
    print(f"  FCP:                 {baseline_metrics.get('first_contentful_paint', 0) or 0:.0f}ms")
    print(f"  LCP:                 {baseline_metrics.get('largest_contentful_paint', 0) or 0:.0f}ms")
    print(f"  TBT:                 {baseline_metrics.get('total_blocking_time', 0) or 0:.0f}ms")
    print(f"  CLS:                 {baseline_metrics.get('cumulative_layout_shift', 0) or 0:.3f}")
    print(f"  Speed Index:         {baseline_metrics.get('speed_index', 0) or 0:.0f}ms")

    # Check for regressions
    has_regression, regression_messages = check_regression(
        current_metrics,
        baseline_metrics,
        args.threshold
    )

    # Print regression results
    print("\n" + "=" * 80)
    if has_regression:
        print("REGRESSION DETECTED!")
        print("=" * 80)
        for msg in regression_messages:
            print(f"  {msg}")
        print("=" * 80)
        print("\nAction Required: Investigate performance degradation")
        return 1
    else:
        print("NO REGRESSION DETECTED")
        print("=" * 80)
        print("\nAll metrics within acceptable threshold")
        return 0


if __name__ == '__main__':
    sys.exit(main())