Download src/eval/cli.ts from Rachit-Tw/PRININIT: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/Rachit-Tw/PRININIT/resolve/main/src/eval/cli.ts
- Command line
-
hf download hf://spaces/Rachit-Tw/PRININIT/src/eval/cli.ts
-
curl -L -o cli.ts https://huggingface.co/spaces/Rachit-Tw/PRININIT/resolve/main/src/eval/cli.ts
10 kB
| /** | |
| * PRIX AI Evaluation System - CLI | |
| * Command-line interface for running evaluations | |
| */ | |
| import * as fs from 'fs' | |
| import * as path from 'path' | |
| import {TestRunner, runTestCase} from './test-runner' | |
| import {Evaluator} from './evaluator' | |
| import {MetricsAggregator} from './metrics' | |
| import {TestResult} from './types' | |
| interface CLIOptions { | |
| command: 'eval' | 'single' | 'compare' | 'baseline' | 'report' | |
| testPath?: string | |
| testsDir: string | |
| outputDir: string | |
| verbose: boolean | |
| format: 'text' | 'json' | 'csv' | 'markdown' | |
| failOnRegression: boolean | |
| } | |
| const DEFAULT_TESTS_DIR = path.join(process.cwd(), 'tests') | |
| const DEFAULT_OUTPUT_DIR = path.join(process.cwd(), 'eval-output') | |
| const BASELINE_PATH = path.join(DEFAULT_OUTPUT_DIR, 'baseline_metrics.json') | |
| const HISTORY_PATH = path.join(DEFAULT_OUTPUT_DIR, 'metrics_history.json') | |
| function parseArgs(): CLIOptions { | |
| const args = process.argv.slice(2) | |
| const options: CLIOptions = { | |
| command: 'eval', | |
| testsDir: DEFAULT_TESTS_DIR, | |
| outputDir: DEFAULT_OUTPUT_DIR, | |
| verbose: false, | |
| format: 'text', | |
| failOnRegression: false | |
| } | |
| for (let i = 0; i < args.length; i++) { | |
| const arg = args[i] | |
| switch (arg) { | |
| case 'eval': | |
| case 'single': | |
| case 'compare': | |
| case 'baseline': | |
| case 'report': | |
| options.command = arg | |
| break | |
| case '--test': | |
| case '-t': | |
| options.testPath = args[++i] | |
| break | |
| case '--tests-dir': | |
| case '-d': | |
| options.testsDir = args[++i] | |
| break | |
| case '--output': | |
| case '-o': | |
| options.outputDir = args[++i] | |
| break | |
| case '--verbose': | |
| case '-v': | |
| options.verbose = true | |
| break | |
| case '--format': | |
| case '-f': | |
| options.format = args[++i] as any | |
| break | |
| case '--fail-on-regression': | |
| options.failOnRegression = true | |
| break | |
| case '--help': | |
| case '-h': | |
| printHelp() | |
| process.exit(0) | |
| } | |
| } | |
| return options | |
| } | |
| function printHelp(): void { | |
| console.log(` | |
| PRIX AI Evaluation System | |
| Usage: npm run test:eval [command] [options] | |
| Commands: | |
| eval Run all test cases (default) | |
| single Run a single test case (--test required) | |
| compare Compare current results with baseline | |
| baseline Save current results as new baseline | |
| report Generate report from last run | |
| Options: | |
| -t, --test <path> Path to specific test case | |
| -d, --tests-dir <path> Directory containing tests (default: ./tests) | |
| -o, --output <path> Output directory (default: ./eval-output) | |
| -v, --verbose Enable verbose logging | |
| -f, --format <format> Output format: text, json, csv, markdown | |
| --fail-on-regression Exit with error if regression detected | |
| -h, --help Show this help | |
| Examples: | |
| npm run test:eval # Run all tests | |
| npm run test:eval single -t tests/bugs/mutation # Run single test | |
| npm run test:eval compare # Compare with baseline | |
| npm run test:eval baseline # Save new baseline | |
| npm run test:eval -f json # Output as JSON | |
| `) | |
| } | |
| async function runAllTests(options: CLIOptions): Promise<void> { | |
| console.log('π PRIX AI Evaluation System') | |
| console.log(`π Tests directory: ${options.testsDir}`) | |
| console.log(`π Output directory: ${options.outputDir}`) | |
| console.log('') | |
| const runner = new TestRunner({ | |
| testsDir: options.testsDir, | |
| outputDir: options.outputDir, | |
| verbose: options.verbose, | |
| skipAutoPR: true | |
| }) | |
| const testCases = runner.loadTestCases() | |
| if (testCases.length === 0) { | |
| console.error('β No test cases found!') | |
| process.exit(1) | |
| } | |
| console.log(`π Found ${testCases.length} test case(s)`) | |
| console.log('') | |
| const evaluator = new Evaluator() | |
| const results: TestResult[] = [] | |
| for (const testCase of testCases) { | |
| const output = await runner.runTestCase(testCase) | |
| const result = evaluator.evaluate(testCase, output) | |
| results.push(result) | |
| // Print individual result | |
| if (options.format === 'text') { | |
| console.log(evaluator.generateReport(result)) | |
| } | |
| } | |
| // Aggregate metrics | |
| const metrics = new MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }) | |
| const summary = metrics.aggregate(results) | |
| const regression = metrics.detectRegression(results) | |
| // Save results | |
| const resultsPath = path.join(options.outputDir, 'last_results.json') | |
| fs.writeFileSync(resultsPath, JSON.stringify({results, summary}, null, 2)) | |
| // Append to history | |
| metrics.appendHistory(results) | |
| // Generate and print report | |
| if (options.format === 'text') { | |
| console.log(metrics.generateReport(summary, regression)) | |
| } else { | |
| console.log(metrics.exportMetrics(summary, options.format)) | |
| } | |
| // Exit with error if regression detected and flag is set | |
| if (options.failOnRegression && regression.hasRegression) { | |
| console.error('\nβ REGRESSION DETECTED - Exiting with error') | |
| process.exit(1) | |
| } | |
| // Exit with error if any tests failed | |
| if (summary.failedTests > 0) { | |
| process.exit(1) | |
| } | |
| } | |
| async function runSingleTest(options: CLIOptions): Promise<void> { | |
| if (!options.testPath) { | |
| console.error('β --test path is required for single test mode') | |
| process.exit(1) | |
| } | |
| const fullPath = path.resolve(options.testPath) | |
| if (!fs.existsSync(fullPath)) { | |
| console.error(`β Test path not found: ${fullPath}`) | |
| process.exit(1) | |
| } | |
| console.log(`π§ͺ Running single test: ${fullPath}`) | |
| const output = await runTestCase(fullPath, { | |
| verbose: options.verbose, | |
| skipAutoPR: true | |
| }) | |
| const expectedPath = path.join(fullPath, 'expected.json') | |
| const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8')) | |
| const evaluator = new Evaluator() | |
| const testCase = { | |
| name: path.basename(fullPath), | |
| path: fullPath, | |
| category: path.basename(path.dirname(fullPath)), | |
| basePath: path.join(fullPath, 'base'), | |
| headPath: path.join(fullPath, 'head'), | |
| expected | |
| } | |
| const result = evaluator.evaluate(testCase, output) | |
| if (options.format === 'text') { | |
| console.log(evaluator.generateReport(result)) | |
| } else { | |
| console.log(JSON.stringify(result, null, 2)) | |
| } | |
| if (!result.passed) { | |
| process.exit(1) | |
| } | |
| } | |
| async function compareWithBaseline(options: CLIOptions): Promise<void> { | |
| const resultsPath = path.join(options.outputDir, 'last_results.json') | |
| if (!fs.existsSync(resultsPath)) { | |
| console.error('β No results found. Run tests first.') | |
| process.exit(1) | |
| } | |
| const {results} = JSON.parse(fs.readFileSync(resultsPath, 'utf8')) | |
| const metrics = new MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }) | |
| const regression = metrics.detectRegression(results) | |
| console.log('π Regression Analysis') | |
| console.log('='.repeat(50)) | |
| console.log( | |
| `Status: ${ | |
| regression.hasRegression ? 'β REGRESSION DETECTED' : 'β No Regression' | |
| }` | |
| ) | |
| console.log(`Accuracy Delta: ${(regression.accuracyDelta * 100).toFixed(2)}%`) | |
| if (regression.newFailures.length > 0) { | |
| console.log('\nNew Failures:') | |
| for (const test of regression.newFailures) { | |
| console.log(` β ${test}`) | |
| } | |
| } | |
| if (regression.fixedTests.length > 0) { | |
| console.log('\nFixed Tests:') | |
| for (const test of regression.fixedTests) { | |
| console.log(` β ${test}`) | |
| } | |
| } | |
| if (options.failOnRegression && regression.hasRegression) { | |
| process.exit(1) | |
| } | |
| } | |
| async function saveBaseline(options: CLIOptions): Promise<void> { | |
| const resultsPath = path.join(options.outputDir, 'last_results.json') | |
| if (!fs.existsSync(resultsPath)) { | |
| console.error('β No results found. Run tests first.') | |
| process.exit(1) | |
| } | |
| const {results} = JSON.parse(fs.readFileSync(resultsPath, 'utf8')) | |
| const metrics = new MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }) | |
| metrics.saveBaseline(results) | |
| console.log('β Baseline saved successfully') | |
| } | |
| async function generateReport(options: CLIOptions): Promise<void> { | |
| const resultsPath = path.join(options.outputDir, 'last_results.json') | |
| if (!fs.existsSync(resultsPath)) { | |
| console.error('β No results found. Run tests first.') | |
| process.exit(1) | |
| } | |
| const {summary} = JSON.parse(fs.readFileSync(resultsPath, 'utf8')) | |
| const metrics = new MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }) | |
| const regression = metrics.detectRegression([]) | |
| if (options.format === 'text') { | |
| console.log(metrics.generateReport(summary, regression)) | |
| } else { | |
| console.log(metrics.exportMetrics(summary, options.format)) | |
| } | |
| } | |
| async function main(): Promise<void> { | |
| const options = parseArgs() | |
| try { | |
| switch (options.command) { | |
| case 'eval': | |
| await runAllTests(options) | |
| break | |
| case 'single': | |
| await runSingleTest(options) | |
| break | |
| case 'compare': | |
| await compareWithBaseline(options) | |
| break | |
| case 'baseline': | |
| await saveBaseline(options) | |
| break | |
| case 'report': | |
| await generateReport(options) | |
| break | |
| } | |
| } catch (err: any) { | |
| console.error(`β Error: ${err.message}`) | |
| if (options.verbose) { | |
| console.error(err.stack) | |
| } | |
| process.exit(1) | |
| } | |
| } | |
| main() | |