Download lib/src/eval/cli.js from Rachit-Tw/PRININIT: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/spaces/Rachit-Tw/PRININIT/resolve/main/lib/src/eval/cli.js
- Command line
-
hf download hf://spaces/Rachit-Tw/PRININIT/lib/src/eval/cli.js
-
curl -L -o cli.js https://huggingface.co/spaces/Rachit-Tw/PRININIT/resolve/main/lib/src/eval/cli.js
11.7 kB
| ; | |
| /** | |
| * PRIX AI Evaluation System - CLI | |
| * Command-line interface for running evaluations | |
| */ | |
| var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { | |
| if (k2 === undefined) k2 = k; | |
| var desc = Object.getOwnPropertyDescriptor(m, k); | |
| if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { | |
| desc = { enumerable: true, get: function() { return m[k]; } }; | |
| } | |
| Object.defineProperty(o, k2, desc); | |
| }) : (function(o, m, k, k2) { | |
| if (k2 === undefined) k2 = k; | |
| o[k2] = m[k]; | |
| })); | |
| var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { | |
| Object.defineProperty(o, "default", { enumerable: true, value: v }); | |
| }) : function(o, v) { | |
| o["default"] = v; | |
| }); | |
| var __importStar = (this && this.__importStar) || (function () { | |
| var ownKeys = function(o) { | |
| ownKeys = Object.getOwnPropertyNames || function (o) { | |
| var ar = []; | |
| for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; | |
| return ar; | |
| }; | |
| return ownKeys(o); | |
| }; | |
| return function (mod) { | |
| if (mod && mod.__esModule) return mod; | |
| var result = {}; | |
| if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); | |
| __setModuleDefault(result, mod); | |
| return result; | |
| }; | |
| })(); | |
| Object.defineProperty(exports, "__esModule", { value: true }); | |
| const fs = __importStar(require("fs")); | |
| const path = __importStar(require("path")); | |
| const test_runner_1 = require("./test-runner"); | |
| const evaluator_1 = require("./evaluator"); | |
| const metrics_1 = require("./metrics"); | |
| const DEFAULT_TESTS_DIR = path.join(process.cwd(), 'tests'); | |
| const DEFAULT_OUTPUT_DIR = path.join(process.cwd(), 'eval-output'); | |
| const BASELINE_PATH = path.join(DEFAULT_OUTPUT_DIR, 'baseline_metrics.json'); | |
| const HISTORY_PATH = path.join(DEFAULT_OUTPUT_DIR, 'metrics_history.json'); | |
| function parseArgs() { | |
| const args = process.argv.slice(2); | |
| const options = { | |
| command: 'eval', | |
| testsDir: DEFAULT_TESTS_DIR, | |
| outputDir: DEFAULT_OUTPUT_DIR, | |
| verbose: false, | |
| format: 'text', | |
| failOnRegression: false | |
| }; | |
| for (let i = 0; i < args.length; i++) { | |
| const arg = args[i]; | |
| switch (arg) { | |
| case 'eval': | |
| case 'single': | |
| case 'compare': | |
| case 'baseline': | |
| case 'report': | |
| options.command = arg; | |
| break; | |
| case '--test': | |
| case '-t': | |
| options.testPath = args[++i]; | |
| break; | |
| case '--tests-dir': | |
| case '-d': | |
| options.testsDir = args[++i]; | |
| break; | |
| case '--output': | |
| case '-o': | |
| options.outputDir = args[++i]; | |
| break; | |
| case '--verbose': | |
| case '-v': | |
| options.verbose = true; | |
| break; | |
| case '--format': | |
| case '-f': | |
| options.format = args[++i]; | |
| break; | |
| case '--fail-on-regression': | |
| options.failOnRegression = true; | |
| break; | |
| case '--help': | |
| case '-h': | |
| printHelp(); | |
| process.exit(0); | |
| } | |
| } | |
| return options; | |
| } | |
| function printHelp() { | |
| console.log(` | |
| PRIX AI Evaluation System | |
| Usage: npm run test:eval [command] [options] | |
| Commands: | |
| eval Run all test cases (default) | |
| single Run a single test case (--test required) | |
| compare Compare current results with baseline | |
| baseline Save current results as new baseline | |
| report Generate report from last run | |
| Options: | |
| -t, --test <path> Path to specific test case | |
| -d, --tests-dir <path> Directory containing tests (default: ./tests) | |
| -o, --output <path> Output directory (default: ./eval-output) | |
| -v, --verbose Enable verbose logging | |
| -f, --format <format> Output format: text, json, csv, markdown | |
| --fail-on-regression Exit with error if regression detected | |
| -h, --help Show this help | |
| Examples: | |
| npm run test:eval # Run all tests | |
| npm run test:eval single -t tests/bugs/mutation # Run single test | |
| npm run test:eval compare # Compare with baseline | |
| npm run test:eval baseline # Save new baseline | |
| npm run test:eval -f json # Output as JSON | |
| `); | |
| } | |
| async function runAllTests(options) { | |
| console.log('π PRIX AI Evaluation System'); | |
| console.log(`π Tests directory: ${options.testsDir}`); | |
| console.log(`π Output directory: ${options.outputDir}`); | |
| console.log(''); | |
| const runner = new test_runner_1.TestRunner({ | |
| testsDir: options.testsDir, | |
| outputDir: options.outputDir, | |
| verbose: options.verbose, | |
| skipAutoPR: true | |
| }); | |
| const testCases = runner.loadTestCases(); | |
| if (testCases.length === 0) { | |
| console.error('β No test cases found!'); | |
| process.exit(1); | |
| } | |
| console.log(`π Found ${testCases.length} test case(s)`); | |
| console.log(''); | |
| const evaluator = new evaluator_1.Evaluator(); | |
| const results = []; | |
| for (const testCase of testCases) { | |
| const output = await runner.runTestCase(testCase); | |
| const result = evaluator.evaluate(testCase, output); | |
| results.push(result); | |
| // Print individual result | |
| if (options.format === 'text') { | |
| console.log(evaluator.generateReport(result)); | |
| } | |
| } | |
| // Aggregate metrics | |
| const metrics = new metrics_1.MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }); | |
| const summary = metrics.aggregate(results); | |
| const regression = metrics.detectRegression(results); | |
| // Save results | |
| const resultsPath = path.join(options.outputDir, 'last_results.json'); | |
| fs.writeFileSync(resultsPath, JSON.stringify({ results, summary }, null, 2)); | |
| // Append to history | |
| metrics.appendHistory(results); | |
| // Generate and print report | |
| if (options.format === 'text') { | |
| console.log(metrics.generateReport(summary, regression)); | |
| } | |
| else { | |
| console.log(metrics.exportMetrics(summary, options.format)); | |
| } | |
| // Exit with error if regression detected and flag is set | |
| if (options.failOnRegression && regression.hasRegression) { | |
| console.error('\nβ REGRESSION DETECTED - Exiting with error'); | |
| process.exit(1); | |
| } | |
| // Exit with error if any tests failed | |
| if (summary.failedTests > 0) { | |
| process.exit(1); | |
| } | |
| } | |
| async function runSingleTest(options) { | |
| if (!options.testPath) { | |
| console.error('β --test path is required for single test mode'); | |
| process.exit(1); | |
| } | |
| const fullPath = path.resolve(options.testPath); | |
| if (!fs.existsSync(fullPath)) { | |
| console.error(`β Test path not found: ${fullPath}`); | |
| process.exit(1); | |
| } | |
| console.log(`π§ͺ Running single test: ${fullPath}`); | |
| const output = await (0, test_runner_1.runTestCase)(fullPath, { | |
| verbose: options.verbose, | |
| skipAutoPR: true | |
| }); | |
| const expectedPath = path.join(fullPath, 'expected.json'); | |
| const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8')); | |
| const evaluator = new evaluator_1.Evaluator(); | |
| const testCase = { | |
| name: path.basename(fullPath), | |
| path: fullPath, | |
| category: path.basename(path.dirname(fullPath)), | |
| basePath: path.join(fullPath, 'base'), | |
| headPath: path.join(fullPath, 'head'), | |
| expected | |
| }; | |
| const result = evaluator.evaluate(testCase, output); | |
| if (options.format === 'text') { | |
| console.log(evaluator.generateReport(result)); | |
| } | |
| else { | |
| console.log(JSON.stringify(result, null, 2)); | |
| } | |
| if (!result.passed) { | |
| process.exit(1); | |
| } | |
| } | |
| async function compareWithBaseline(options) { | |
| const resultsPath = path.join(options.outputDir, 'last_results.json'); | |
| if (!fs.existsSync(resultsPath)) { | |
| console.error('β No results found. Run tests first.'); | |
| process.exit(1); | |
| } | |
| const { results } = JSON.parse(fs.readFileSync(resultsPath, 'utf8')); | |
| const metrics = new metrics_1.MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }); | |
| const regression = metrics.detectRegression(results); | |
| console.log('π Regression Analysis'); | |
| console.log('='.repeat(50)); | |
| console.log(`Status: ${regression.hasRegression ? 'β REGRESSION DETECTED' : 'β No Regression'}`); | |
| console.log(`Accuracy Delta: ${(regression.accuracyDelta * 100).toFixed(2)}%`); | |
| if (regression.newFailures.length > 0) { | |
| console.log('\nNew Failures:'); | |
| for (const test of regression.newFailures) { | |
| console.log(` β ${test}`); | |
| } | |
| } | |
| if (regression.fixedTests.length > 0) { | |
| console.log('\nFixed Tests:'); | |
| for (const test of regression.fixedTests) { | |
| console.log(` β ${test}`); | |
| } | |
| } | |
| if (options.failOnRegression && regression.hasRegression) { | |
| process.exit(1); | |
| } | |
| } | |
| async function saveBaseline(options) { | |
| const resultsPath = path.join(options.outputDir, 'last_results.json'); | |
| if (!fs.existsSync(resultsPath)) { | |
| console.error('β No results found. Run tests first.'); | |
| process.exit(1); | |
| } | |
| const { results } = JSON.parse(fs.readFileSync(resultsPath, 'utf8')); | |
| const metrics = new metrics_1.MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }); | |
| metrics.saveBaseline(results); | |
| console.log('β Baseline saved successfully'); | |
| } | |
| async function generateReport(options) { | |
| const resultsPath = path.join(options.outputDir, 'last_results.json'); | |
| if (!fs.existsSync(resultsPath)) { | |
| console.error('β No results found. Run tests first.'); | |
| process.exit(1); | |
| } | |
| const { summary } = JSON.parse(fs.readFileSync(resultsPath, 'utf8')); | |
| const metrics = new metrics_1.MetricsAggregator({ | |
| baselinePath: path.join(options.outputDir, 'baseline_metrics.json'), | |
| historyPath: path.join(options.outputDir, 'metrics_history.json') | |
| }); | |
| const regression = metrics.detectRegression([]); | |
| if (options.format === 'text') { | |
| console.log(metrics.generateReport(summary, regression)); | |
| } | |
| else { | |
| console.log(metrics.exportMetrics(summary, options.format)); | |
| } | |
| } | |
| async function main() { | |
| const options = parseArgs(); | |
| try { | |
| switch (options.command) { | |
| case 'eval': | |
| await runAllTests(options); | |
| break; | |
| case 'single': | |
| await runSingleTest(options); | |
| break; | |
| case 'compare': | |
| await compareWithBaseline(options); | |
| break; | |
| case 'baseline': | |
| await saveBaseline(options); | |
| break; | |
| case 'report': | |
| await generateReport(options); | |
| break; | |
| } | |
| } | |
| catch (err) { | |
| console.error(`β Error: ${err.message}`); | |
| if (options.verbose) { | |
| console.error(err.stack); | |
| } | |
| process.exit(1); | |
| } | |
| } | |
| main(); | |
| //# sourceMappingURL=cli.js.map |