Buckets:

download
raw
12.4 kB
#!/usr/bin/env python3
"""
Demonstration of the comprehensive validation framework for treesitter-chunker.
This demo shows how to use the validation framework for:
1. Performance validation and benchmarking
2. Load testing and stress testing
3. Regression detection and prevention
4. Full system validation
The validation framework ensures system quality and performance standards.
"""
import sys
import time
from pathlib import Path
# Add the project root to Python path
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from chunker.validation.validation_framework import (
LoadTester,
LoadTestScenario,
PerformanceValidator,
RegressionTester,
ValidationManager,
)
def demo_validation_manager():
"""Demonstrate the ValidationManager orchestration capabilities."""
print("\n" + "=" * 60)
print("VALIDATION MANAGER DEMO")
print("=" * 60)
# Initialize validation manager
validation_manager = ValidationManager()
print("๐Ÿ” Running full validation suite...")
# Configure load tester for quick demo
for scenario in validation_manager.load_tester.test_scenarios.values():
scenario.duration_seconds = 1.0 # Quick demo
scenario.load_levels = [1, 2] # Minimal load levels
# Run full validation
start_time = time.time()
validation_result = validation_manager.run_full_validation()
execution_time = time.time() - start_time
print(f"โœ… Validation completed in {execution_time:.2f} seconds")
print(f"๐Ÿ“Š Validation ID: {validation_result['validation_id']}")
print(f"๐ŸŽฏ Overall Status: {validation_result['status']}")
# Display summary
summary = validation_result["summary"]
print("\n๐Ÿ“ˆ Test Summary:")
print(f" Total Tests: {summary['total_tests']}")
print(f" Passed: {summary['passed']}")
print(f" Failed: {summary['failed']}")
print(f" Errors: {summary['errors']}")
# Display recommendations
print("\n๐Ÿ’ก Recommendations:")
for i, rec in enumerate(validation_result["recommendations"], 1):
print(f" {i}. {rec}")
return validation_result
def demo_performance_validator():
"""Demonstrate performance validation capabilities."""
print("\n" + "=" * 60)
print("PERFORMANCE VALIDATOR DEMO")
print("=" * 60)
# Initialize performance validator
performance_validator = PerformanceValidator()
print("๐Ÿ”ง Running performance benchmark validation...")
# Run performance benchmarks
benchmark_result = performance_validator.validate_performance_benchmarks()
print(f"โœ… Benchmark validation: {benchmark_result['status']}")
print(f"๐Ÿฅ Overall health: {benchmark_result['overall_health']}")
print(
f"๐Ÿ“Š Optimization potential: {benchmark_result['optimization_potential']:.1f}%",
)
# Display test results
test_results = benchmark_result["test_results"]
print("\n๐Ÿงช Individual Test Results:")
for result in test_results:
status_icon = "โœ…" if result["status"] == "passed" else "โŒ"
print(f" {status_icon} {result['test_name']}: {result['status']}")
if "execution_time" in result:
print(f" โฑ๏ธ Execution time: {result['execution_time']:.3f}ms")
print("๐Ÿ” Running SLA validation...")
# Run SLA validation
sla_result = performance_validator.validate_performance_sla()
print(f"โœ… SLA validation: {sla_result['status']}")
print(f"๐Ÿ“‹ SLA compliance: {sla_result.get('sla_compliance', 'Unknown')}")
# Generate performance report
print("๐Ÿ“„ Generating comprehensive performance report...")
report = performance_validator.generate_performance_report()
print(f"๐Ÿ“Š Overall performance status: {report['overall_status']}")
return benchmark_result, sla_result, report
def demo_load_tester():
"""Demonstrate load testing capabilities."""
print("\n" + "=" * 60)
print("LOAD TESTER DEMO")
print("=" * 60)
# Initialize load tester
load_tester = LoadTester()
# Configure for quick demo
for scenario in load_tester.test_scenarios.values():
scenario.duration_seconds = 2.0 # Quick demo
scenario.load_levels = [1, 3] # Minimal load levels
print("๐Ÿš€ Running individual load test...")
# Run a single load test
load_result = load_tester.run_load_test("basic_processing", 2)
print(f"โœ… Load test: {load_result['status']}")
print(f"๐Ÿ“Š Scenario: {load_result['scenario']}")
print(f"๐ŸŽฏ Load level: {load_result['load_level']}")
metrics = load_result["metrics"]
print("\n๐Ÿ“ˆ Performance Metrics:")
print(f" Success rate: {metrics['success_rate_percent']:.1f}%")
print(f" Average response time: {metrics['average_response_time_ms']:.2f}ms")
print(f" Throughput: {metrics['throughput_requests_per_sec']:.1f} req/s")
print(f" Total requests: {metrics['total_requests']}")
print("\n๐Ÿ”ฅ Running stress test...")
# Run stress test
stress_result = load_tester.run_stress_test("basic_processing")
print(f"โœ… Stress test: {stress_result['status']}")
print(
f"๐Ÿ’ช Breaking point: {stress_result['breaking_point_load_level']} concurrent requests",
)
print(f"๐Ÿ”ฌ Tests run: {stress_result['total_tests_run']}")
# Add custom test scenario
print("\n๐ŸŽ›๏ธ Adding custom test scenario...")
def custom_test_function(data=None):
"""Custom test function for demo."""
time.sleep(0.01) # Simulate 10ms work
return {"result": "custom_success", "data_processed": bool(data)}
custom_scenario = LoadTestScenario(
name="custom_demo",
description="Custom demo scenario",
target_function=custom_test_function,
load_levels=[1, 2],
duration_seconds=1.0,
ramp_up_seconds=0.1,
success_criteria={"max_response_time_ms": 100, "min_success_rate_percent": 95},
test_data_generator=lambda: "demo_data",
)
load_tester.add_test_scenario(custom_scenario)
# Test custom scenario
custom_result = load_tester.run_load_test("custom_demo", 1)
print(f"โœ… Custom scenario test: {custom_result['status']}")
return load_result, stress_result, custom_result
def demo_regression_tester():
"""Demonstrate regression testing capabilities."""
print("\n" + "=" * 60)
print("REGRESSION TESTER DEMO")
print("=" * 60)
# Initialize regression tester
regression_tester = RegressionTester()
print("๐Ÿ“ Establishing performance baseline...")
# Establish baseline
baseline_result = regression_tester.establish_baseline()
print(f"โœ… Baseline establishment: {baseline_result['status']}")
print(f"๐Ÿ“Š Metrics captured: {baseline_result['metrics_count']}")
print(f"๐Ÿงช Benchmark tests: {baseline_result['benchmark_tests']}")
# Wait a moment to ensure different timestamp
time.sleep(0.1)
print("\n๐Ÿ” Detecting performance regressions...")
# Detect regressions
regression_result = regression_tester.detect_regressions()
print(f"โœ… Regression detection: {regression_result['status']}")
print(f"โš ๏ธ Regressions detected: {regression_result['regressions_detected']}")
if regression_result["regressions"]:
print("\n๐Ÿšจ Regression Details:")
for regression in regression_result["regressions"]:
print(
f" โ€ข {regression.get('metric_name', regression.get('benchmark_name', 'Unknown'))}",
)
print(f" Change: {regression['change_percent']:.1f}%")
print(f" Severity: {regression['severity']}")
print("\n๐Ÿ“Š Assessing change impact...")
# Assess change impact
test_changes = [
"memory optimization update",
"algorithm performance improvement",
"ui color scheme change",
]
change_impact = regression_tester.assess_change_impact(test_changes)
print(f"โœ… Change impact assessment: {change_impact['status']}")
print(f"๐ŸŽฏ Overall risk level: {change_impact['overall_risk_level']}")
print(f"โš ๏ธ High-risk changes: {change_impact['high_risk_changes']}")
print("\n๐Ÿ›ก๏ธ Implementing regression prevention...")
# Implement regression prevention
prevention_result = regression_tester.prevent_regressions()
print(f"โœ… Prevention implementation: {prevention_result['status']}")
print(
f"๐Ÿ”ง Measures implemented: {prevention_result['measures_implemented']}/{prevention_result['total_measures']}",
)
print(f"๐Ÿ“ˆ Success rate: {prevention_result['success_rate']:.1f}%")
return baseline_result, regression_result, change_impact, prevention_result
def demo_integration_workflow():
"""Demonstrate a complete validation workflow."""
print("\n" + "=" * 60)
print("INTEGRATION WORKFLOW DEMO")
print("=" * 60)
print("๐Ÿ”„ Running complete validation workflow...")
# Initialize all components
validation_manager = ValidationManager()
# Configure for quick demo
for scenario in validation_manager.load_tester.test_scenarios.values():
scenario.duration_seconds = 1.0
scenario.load_levels = [1, 2]
# Step 1: Establish baseline for future comparisons
print("\n1๏ธโƒฃ Establishing performance baseline...")
baseline = validation_manager.regression_tester.establish_baseline()
print(f" โœ… Baseline: {baseline['status']}")
# Step 2: Run performance validation
print("\n2๏ธโƒฃ Validating current performance...")
performance = validation_manager.validate_performance()
print(f" โœ… Performance: {performance['status']}")
# Step 3: Execute load tests
print("\n3๏ธโƒฃ Executing load tests...")
load_tests = validation_manager.run_load_tests()
print(f" โœ… Load tests: {load_tests['status']}")
# Step 4: Check for regressions
print("\n4๏ธโƒฃ Checking for regressions...")
regressions = validation_manager.run_regression_tests()
print(f" โœ… Regressions: {regressions['status']}")
# Step 5: Generate comprehensive report
print("\n5๏ธโƒฃ Generating validation report...")
report = validation_manager.generate_validation_report()
print("\n๐Ÿ“‹ Final Validation Report:")
print(f" Status: {report['status']}")
if "summary" in report:
summary = report["summary"]
print(f" Tests: {summary['passed']}/{summary['total_tests']} passed")
print("\n๐Ÿ’ก Final Recommendations:")
for rec in report.get(
"recommendations",
["System validation completed successfully"],
):
print(f" โ€ข {rec}")
return report
def main():
"""Run the complete validation framework demonstration."""
print("๐Ÿš€ TREESITTER-CHUNKER VALIDATION FRAMEWORK DEMO")
print("๐Ÿ” Comprehensive system validation and testing capabilities")
try:
# Demo each component
validation_result = demo_validation_manager()
performance_results = demo_performance_validator()
load_results = demo_load_tester()
regression_results = demo_regression_tester()
# Demo integration workflow
integration_result = demo_integration_workflow()
print("\n" + "=" * 60)
print("DEMO COMPLETED SUCCESSFULLY! ๐ŸŽ‰")
print("=" * 60)
print("\nโœ… All validation framework components demonstrated:")
print(" ๐ŸŽฏ ValidationManager - Orchestration and coordination")
print(" ๐Ÿ“Š PerformanceValidator - Benchmark and SLA validation")
print(" ๐Ÿš€ LoadTester - Load and stress testing")
print(" ๐Ÿ” RegressionTester - Regression detection and prevention")
print(" ๐Ÿ”„ Integration workflow - Complete validation pipeline")
print("\n๐Ÿ† The validation framework provides:")
print(" โ€ข Comprehensive performance monitoring")
print(" โ€ข Automated load and stress testing")
print(" โ€ข Regression detection and prevention")
print(" โ€ข Quality assurance and system validation")
print(" โ€ข Production-ready testing capabilities")
except Exception as e:
print(f"\nโŒ Demo failed with error: {e}")
import traceback
traceback.print_exc()
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

Xet Storage Details

Size:
12.4 kB
ยท
Xet hash:
11a2ac31bb99b2de64092b53885d1ca9075ce6f5c6d98f2ed0bd903ce72cf88e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.