Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /validation_framework_demo.py
| #!/usr/bin/env python3 | |
| """ | |
| Demonstration of the comprehensive validation framework for treesitter-chunker. | |
| This demo shows how to use the validation framework for: | |
| 1. Performance validation and benchmarking | |
| 2. Load testing and stress testing | |
| 3. Regression detection and prevention | |
| 4. Full system validation | |
| The validation framework ensures system quality and performance standards. | |
| """ | |
| import sys | |
| import time | |
| from pathlib import Path | |
| # Add the project root to Python path | |
| project_root = Path(__file__).parent.parent | |
| sys.path.insert(0, str(project_root)) | |
| from chunker.validation.validation_framework import ( | |
| LoadTester, | |
| LoadTestScenario, | |
| PerformanceValidator, | |
| RegressionTester, | |
| ValidationManager, | |
| ) | |
| def demo_validation_manager(): | |
| """Demonstrate the ValidationManager orchestration capabilities.""" | |
| print("\n" + "=" * 60) | |
| print("VALIDATION MANAGER DEMO") | |
| print("=" * 60) | |
| # Initialize validation manager | |
| validation_manager = ValidationManager() | |
| print("๐ Running full validation suite...") | |
| # Configure load tester for quick demo | |
| for scenario in validation_manager.load_tester.test_scenarios.values(): | |
| scenario.duration_seconds = 1.0 # Quick demo | |
| scenario.load_levels = [1, 2] # Minimal load levels | |
| # Run full validation | |
| start_time = time.time() | |
| validation_result = validation_manager.run_full_validation() | |
| execution_time = time.time() - start_time | |
| print(f"โ Validation completed in {execution_time:.2f} seconds") | |
| print(f"๐ Validation ID: {validation_result['validation_id']}") | |
| print(f"๐ฏ Overall Status: {validation_result['status']}") | |
| # Display summary | |
| summary = validation_result["summary"] | |
| print("\n๐ Test Summary:") | |
| print(f" Total Tests: {summary['total_tests']}") | |
| print(f" Passed: {summary['passed']}") | |
| print(f" Failed: {summary['failed']}") | |
| print(f" Errors: {summary['errors']}") | |
| # Display recommendations | |
| print("\n๐ก Recommendations:") | |
| for i, rec in enumerate(validation_result["recommendations"], 1): | |
| print(f" {i}. {rec}") | |
| return validation_result | |
| def demo_performance_validator(): | |
| """Demonstrate performance validation capabilities.""" | |
| print("\n" + "=" * 60) | |
| print("PERFORMANCE VALIDATOR DEMO") | |
| print("=" * 60) | |
| # Initialize performance validator | |
| performance_validator = PerformanceValidator() | |
| print("๐ง Running performance benchmark validation...") | |
| # Run performance benchmarks | |
| benchmark_result = performance_validator.validate_performance_benchmarks() | |
| print(f"โ Benchmark validation: {benchmark_result['status']}") | |
| print(f"๐ฅ Overall health: {benchmark_result['overall_health']}") | |
| print( | |
| f"๐ Optimization potential: {benchmark_result['optimization_potential']:.1f}%", | |
| ) | |
| # Display test results | |
| test_results = benchmark_result["test_results"] | |
| print("\n๐งช Individual Test Results:") | |
| for result in test_results: | |
| status_icon = "โ " if result["status"] == "passed" else "โ" | |
| print(f" {status_icon} {result['test_name']}: {result['status']}") | |
| if "execution_time" in result: | |
| print(f" โฑ๏ธ Execution time: {result['execution_time']:.3f}ms") | |
| print("๐ Running SLA validation...") | |
| # Run SLA validation | |
| sla_result = performance_validator.validate_performance_sla() | |
| print(f"โ SLA validation: {sla_result['status']}") | |
| print(f"๐ SLA compliance: {sla_result.get('sla_compliance', 'Unknown')}") | |
| # Generate performance report | |
| print("๐ Generating comprehensive performance report...") | |
| report = performance_validator.generate_performance_report() | |
| print(f"๐ Overall performance status: {report['overall_status']}") | |
| return benchmark_result, sla_result, report | |
| def demo_load_tester(): | |
| """Demonstrate load testing capabilities.""" | |
| print("\n" + "=" * 60) | |
| print("LOAD TESTER DEMO") | |
| print("=" * 60) | |
| # Initialize load tester | |
| load_tester = LoadTester() | |
| # Configure for quick demo | |
| for scenario in load_tester.test_scenarios.values(): | |
| scenario.duration_seconds = 2.0 # Quick demo | |
| scenario.load_levels = [1, 3] # Minimal load levels | |
| print("๐ Running individual load test...") | |
| # Run a single load test | |
| load_result = load_tester.run_load_test("basic_processing", 2) | |
| print(f"โ Load test: {load_result['status']}") | |
| print(f"๐ Scenario: {load_result['scenario']}") | |
| print(f"๐ฏ Load level: {load_result['load_level']}") | |
| metrics = load_result["metrics"] | |
| print("\n๐ Performance Metrics:") | |
| print(f" Success rate: {metrics['success_rate_percent']:.1f}%") | |
| print(f" Average response time: {metrics['average_response_time_ms']:.2f}ms") | |
| print(f" Throughput: {metrics['throughput_requests_per_sec']:.1f} req/s") | |
| print(f" Total requests: {metrics['total_requests']}") | |
| print("\n๐ฅ Running stress test...") | |
| # Run stress test | |
| stress_result = load_tester.run_stress_test("basic_processing") | |
| print(f"โ Stress test: {stress_result['status']}") | |
| print( | |
| f"๐ช Breaking point: {stress_result['breaking_point_load_level']} concurrent requests", | |
| ) | |
| print(f"๐ฌ Tests run: {stress_result['total_tests_run']}") | |
| # Add custom test scenario | |
| print("\n๐๏ธ Adding custom test scenario...") | |
| def custom_test_function(data=None): | |
| """Custom test function for demo.""" | |
| time.sleep(0.01) # Simulate 10ms work | |
| return {"result": "custom_success", "data_processed": bool(data)} | |
| custom_scenario = LoadTestScenario( | |
| name="custom_demo", | |
| description="Custom demo scenario", | |
| target_function=custom_test_function, | |
| load_levels=[1, 2], | |
| duration_seconds=1.0, | |
| ramp_up_seconds=0.1, | |
| success_criteria={"max_response_time_ms": 100, "min_success_rate_percent": 95}, | |
| test_data_generator=lambda: "demo_data", | |
| ) | |
| load_tester.add_test_scenario(custom_scenario) | |
| # Test custom scenario | |
| custom_result = load_tester.run_load_test("custom_demo", 1) | |
| print(f"โ Custom scenario test: {custom_result['status']}") | |
| return load_result, stress_result, custom_result | |
| def demo_regression_tester(): | |
| """Demonstrate regression testing capabilities.""" | |
| print("\n" + "=" * 60) | |
| print("REGRESSION TESTER DEMO") | |
| print("=" * 60) | |
| # Initialize regression tester | |
| regression_tester = RegressionTester() | |
| print("๐ Establishing performance baseline...") | |
| # Establish baseline | |
| baseline_result = regression_tester.establish_baseline() | |
| print(f"โ Baseline establishment: {baseline_result['status']}") | |
| print(f"๐ Metrics captured: {baseline_result['metrics_count']}") | |
| print(f"๐งช Benchmark tests: {baseline_result['benchmark_tests']}") | |
| # Wait a moment to ensure different timestamp | |
| time.sleep(0.1) | |
| print("\n๐ Detecting performance regressions...") | |
| # Detect regressions | |
| regression_result = regression_tester.detect_regressions() | |
| print(f"โ Regression detection: {regression_result['status']}") | |
| print(f"โ ๏ธ Regressions detected: {regression_result['regressions_detected']}") | |
| if regression_result["regressions"]: | |
| print("\n๐จ Regression Details:") | |
| for regression in regression_result["regressions"]: | |
| print( | |
| f" โข {regression.get('metric_name', regression.get('benchmark_name', 'Unknown'))}", | |
| ) | |
| print(f" Change: {regression['change_percent']:.1f}%") | |
| print(f" Severity: {regression['severity']}") | |
| print("\n๐ Assessing change impact...") | |
| # Assess change impact | |
| test_changes = [ | |
| "memory optimization update", | |
| "algorithm performance improvement", | |
| "ui color scheme change", | |
| ] | |
| change_impact = regression_tester.assess_change_impact(test_changes) | |
| print(f"โ Change impact assessment: {change_impact['status']}") | |
| print(f"๐ฏ Overall risk level: {change_impact['overall_risk_level']}") | |
| print(f"โ ๏ธ High-risk changes: {change_impact['high_risk_changes']}") | |
| print("\n๐ก๏ธ Implementing regression prevention...") | |
| # Implement regression prevention | |
| prevention_result = regression_tester.prevent_regressions() | |
| print(f"โ Prevention implementation: {prevention_result['status']}") | |
| print( | |
| f"๐ง Measures implemented: {prevention_result['measures_implemented']}/{prevention_result['total_measures']}", | |
| ) | |
| print(f"๐ Success rate: {prevention_result['success_rate']:.1f}%") | |
| return baseline_result, regression_result, change_impact, prevention_result | |
| def demo_integration_workflow(): | |
| """Demonstrate a complete validation workflow.""" | |
| print("\n" + "=" * 60) | |
| print("INTEGRATION WORKFLOW DEMO") | |
| print("=" * 60) | |
| print("๐ Running complete validation workflow...") | |
| # Initialize all components | |
| validation_manager = ValidationManager() | |
| # Configure for quick demo | |
| for scenario in validation_manager.load_tester.test_scenarios.values(): | |
| scenario.duration_seconds = 1.0 | |
| scenario.load_levels = [1, 2] | |
| # Step 1: Establish baseline for future comparisons | |
| print("\n1๏ธโฃ Establishing performance baseline...") | |
| baseline = validation_manager.regression_tester.establish_baseline() | |
| print(f" โ Baseline: {baseline['status']}") | |
| # Step 2: Run performance validation | |
| print("\n2๏ธโฃ Validating current performance...") | |
| performance = validation_manager.validate_performance() | |
| print(f" โ Performance: {performance['status']}") | |
| # Step 3: Execute load tests | |
| print("\n3๏ธโฃ Executing load tests...") | |
| load_tests = validation_manager.run_load_tests() | |
| print(f" โ Load tests: {load_tests['status']}") | |
| # Step 4: Check for regressions | |
| print("\n4๏ธโฃ Checking for regressions...") | |
| regressions = validation_manager.run_regression_tests() | |
| print(f" โ Regressions: {regressions['status']}") | |
| # Step 5: Generate comprehensive report | |
| print("\n5๏ธโฃ Generating validation report...") | |
| report = validation_manager.generate_validation_report() | |
| print("\n๐ Final Validation Report:") | |
| print(f" Status: {report['status']}") | |
| if "summary" in report: | |
| summary = report["summary"] | |
| print(f" Tests: {summary['passed']}/{summary['total_tests']} passed") | |
| print("\n๐ก Final Recommendations:") | |
| for rec in report.get( | |
| "recommendations", | |
| ["System validation completed successfully"], | |
| ): | |
| print(f" โข {rec}") | |
| return report | |
| def main(): | |
| """Run the complete validation framework demonstration.""" | |
| print("๐ TREESITTER-CHUNKER VALIDATION FRAMEWORK DEMO") | |
| print("๐ Comprehensive system validation and testing capabilities") | |
| try: | |
| # Demo each component | |
| validation_result = demo_validation_manager() | |
| performance_results = demo_performance_validator() | |
| load_results = demo_load_tester() | |
| regression_results = demo_regression_tester() | |
| # Demo integration workflow | |
| integration_result = demo_integration_workflow() | |
| print("\n" + "=" * 60) | |
| print("DEMO COMPLETED SUCCESSFULLY! ๐") | |
| print("=" * 60) | |
| print("\nโ All validation framework components demonstrated:") | |
| print(" ๐ฏ ValidationManager - Orchestration and coordination") | |
| print(" ๐ PerformanceValidator - Benchmark and SLA validation") | |
| print(" ๐ LoadTester - Load and stress testing") | |
| print(" ๐ RegressionTester - Regression detection and prevention") | |
| print(" ๐ Integration workflow - Complete validation pipeline") | |
| print("\n๐ The validation framework provides:") | |
| print(" โข Comprehensive performance monitoring") | |
| print(" โข Automated load and stress testing") | |
| print(" โข Regression detection and prevention") | |
| print(" โข Quality assurance and system validation") | |
| print(" โข Production-ready testing capabilities") | |
| except Exception as e: | |
| print(f"\nโ Demo failed with error: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| return 1 | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 12.4 kB
- Xet hash:
- 11a2ac31bb99b2de64092b53885d1ca9075ce6f5c6d98f2ed0bd903ce72cf88e
ยท
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.