Download lib/src/eval/evaluator.js from Rachit-Tw/PRININIT: direct link, hf CLI and curl.
- Browser
- Download file 6.83 kB
-
https://huggingface.co/spaces/Rachit-Tw/PRININIT/resolve/main/lib/src/eval/evaluator.js
- Command line
-
hf download hf://spaces/Rachit-Tw/PRININIT/lib/src/eval/evaluator.js
-
curl -L -o evaluator.js https://huggingface.co/spaces/Rachit-Tw/PRININIT/resolve/main/lib/src/eval/evaluator.js
6.83 kB
| ; | |
| /** | |
| * PRIX AI Evaluation System - Evaluation Engine | |
| * Compares test outputs against expected results | |
| */ | |
| Object.defineProperty(exports, "__esModule", { value: true }); | |
| exports.Evaluator = void 0; | |
| exports.evaluate = evaluate; | |
| class Evaluator { | |
| options; | |
| constructor(options = {}) { | |
| this.options = { | |
| strictConfidence: false, | |
| strictIssueType: false, | |
| allowPartialMatches: true, | |
| ...options | |
| }; | |
| } | |
| /** | |
| * Evaluate a single test output against expected results | |
| */ | |
| evaluate(testCase, output) { | |
| const startTime = Date.now(); | |
| const score = this.calculateScore(output, testCase.expected); | |
| return { | |
| testName: testCase.name, | |
| testPath: testCase.path, | |
| category: testCase.category, | |
| passed: score.overall, | |
| score, | |
| output, | |
| expected: testCase.expected, | |
| executionTime: output.executionTime, | |
| timestamp: new Date().toISOString() | |
| }; | |
| } | |
| /** | |
| * Calculate detailed score for a test | |
| */ | |
| calculateScore(output, expected) { | |
| // Detection check | |
| const detectionMatch = expected.shouldDetectIssue === output.detectedIssues; | |
| // Issue type check (if detection expected) | |
| let typeMatch = true; | |
| if (expected.shouldDetectIssue && expected.issueType && output.issueType) { | |
| typeMatch = this.matchIssueType(expected.issueType, output.issueType); | |
| } | |
| // Confidence check | |
| let confidenceMatch = true; | |
| if (expected.expectedConfidence && output.confidence) { | |
| confidenceMatch = this.matchConfidence(expected.expectedConfidence, output.confidence); | |
| } | |
| // Auto-PR check | |
| const autoPRMatch = expected.shouldAutoPR === output.autoPRTriggered; | |
| // Suggestion count check (if specific count expected) | |
| let suggestionCountMatch = true; | |
| if (expected.expectedSuggestions) { | |
| suggestionCountMatch = | |
| expected.expectedSuggestions.length === output.suggestions.length; | |
| } | |
| // Calculate overall score | |
| const allChecks = [ | |
| detectionMatch, | |
| typeMatch, | |
| confidenceMatch, | |
| autoPRMatch, | |
| suggestionCountMatch | |
| ]; | |
| // Weight detection more heavily | |
| const overall = detectionMatch && | |
| (typeMatch || !this.options.strictIssueType) && | |
| (confidenceMatch || !this.options.strictConfidence) && | |
| autoPRMatch; | |
| return { | |
| detection: detectionMatch, | |
| classification: typeMatch, | |
| confidence: confidenceMatch, | |
| autoPR: autoPRMatch, | |
| overall, | |
| details: { | |
| detectionMatch, | |
| typeMatch, | |
| confidenceMatch, | |
| autoPRMatch, | |
| suggestionCountMatch | |
| } | |
| }; | |
| } | |
| /** | |
| * Match issue types with flexibility | |
| */ | |
| matchIssueType(expected, actual) { | |
| if (expected === actual) | |
| return true; | |
| if (!this.options.strictIssueType) { | |
| // Allow some flexibility in classification | |
| const securityTypes = ['security', 'bug']; | |
| const bugTypes = ['bug', 'mutation']; | |
| if (securityTypes.includes(expected) && securityTypes.includes(actual)) | |
| return true; | |
| if (bugTypes.includes(expected) && bugTypes.includes(actual)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| /** | |
| * Match confidence levels | |
| */ | |
| matchConfidence(expected, actual) { | |
| if (expected === actual) | |
| return true; | |
| if (!this.options.strictConfidence) { | |
| // Allow adjacent confidence levels | |
| const levels = ['low', 'medium', 'high']; | |
| const expectedIdx = levels.indexOf(expected); | |
| const actualIdx = levels.indexOf(actual); | |
| // Within 1 level is acceptable | |
| return Math.abs(expectedIdx - actualIdx) <= 1; | |
| } | |
| return false; | |
| } | |
| /** | |
| * Generate detailed evaluation report for a single test | |
| */ | |
| generateReport(result) { | |
| const lines = [ | |
| `\n${'='.repeat(60)}`, | |
| `Test: ${result.testName} (${result.category})`, | |
| `${'='.repeat(60)}`, | |
| `Status: ${result.passed ? 'β PASS' : 'β FAIL'}`, | |
| `Execution Time: ${result.executionTime}ms`, | |
| '', | |
| '--- Expected ---', | |
| ` Detect Issue: ${result.expected.shouldDetectIssue}`, | |
| ` Issue Type: ${result.expected.issueType || 'any'}`, | |
| ` Confidence: ${result.expected.expectedConfidence || 'any'}`, | |
| ` Auto-PR: ${result.expected.shouldAutoPR}`, | |
| '', | |
| '--- Actual ---', | |
| ` Detected: ${result.output.detectedIssues}`, | |
| ` Type: ${result.output.issueType || 'none'}`, | |
| ` Confidence: ${result.output.confidence || 'none'}`, | |
| ` Auto-PR: ${result.output.autoPRTriggered}`, | |
| ` Suggestions: ${result.output.suggestions.length}`, | |
| ` Comments: ${result.output.comments.length}`, | |
| '', | |
| '--- Score Breakdown ---', | |
| ` Detection: ${result.score.detection ? 'β ' : 'β'}`, | |
| ` Classification: ${result.score.classification ? 'β ' : 'β'}`, | |
| ` Confidence: ${result.score.confidence ? 'β ' : 'β'}`, | |
| ` Auto-PR: ${result.score.autoPR ? 'β ' : 'β'}`, | |
| '', | |
| '--- Errors ---', | |
| ...(result.output.errors.length > 0 | |
| ? result.output.errors.map(e => ` β ${e}`) | |
| : [' None']) | |
| ]; | |
| return lines.join('\n'); | |
| } | |
| /** | |
| * Compare two test results for regression detection | |
| */ | |
| compareResults(baseline, current) { | |
| const changes = []; | |
| if (baseline.passed && !current.passed) { | |
| changes.push(`Test ${current.testName} started failing`); | |
| } | |
| if (!baseline.passed && current.passed) { | |
| changes.push(`Test ${current.testName} now passing (improvement)`); | |
| } | |
| if (baseline.score.overall && !current.score.overall) { | |
| changes.push(`Overall score degraded for ${current.testName}`); | |
| } | |
| return { | |
| regressed: changes.some(c => c.includes('failing') || c.includes('degraded')), | |
| changes | |
| }; | |
| } | |
| } | |
| exports.Evaluator = Evaluator; | |
| /** | |
| * Utility function for quick evaluation | |
| */ | |
| function evaluate(output, expected, options) { | |
| const evaluator = new Evaluator(options); | |
| return evaluator.calculateScore(output, expected); | |
| } | |
| //# sourceMappingURL=evaluator.js.map |