import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const manifestPath = path.join(__dirname, "benchlocal.pack.json"); const scenariosPath = path.join(__dirname, "scenarios.json"); function readJsonSafe(filePath, fallback) { try { if (fs.existsSync(filePath)) { const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "").trim(); return JSON.parse(raw); } } catch (err) { console.error(`[IFEval-Hard] Ошибка чтения ${filePath}:`, err.message); } return fallback; } const defaultManifest = { schemaVersion: 1, protocolVersion: 1, id: "ifeval-hard", name: "IFEval-Hard (90 комплексных задач)", version: "1.0.0", description: "Аутентичный сложный IFEval с мульти-композициями: 30 RU, 30 EN и 30 Code-Switching задач (по 2-3 строгих ограничения на промпт)", entry: "./index.mjs", samplingDefaults: { temperature: 0.0 }, capabilities: { tools: false, multiTurn: false, streamingProgress: false, verification: false } }; export const manifest = readJsonSafe(manifestPath, defaultManifest); function loadScenarios() { return readJsonSafe(scenariosPath, []); } export function listScenarios() { return loadScenarios().map(s => ({ id: s.id, name: s.name, category: s.category || "Общие", description: s.name })); } export function getScenarios() { return listScenarios(); } function validateSingleRule(type, args, text) { const clean = (text || "").trim(); const lower = clean.toLowerCase(); switch (type) { case "code_switching": { const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w)); const ruWords = words.filter(w => /[а-яА-ЯёЁ]/.test(w)).length; const enWords = words.filter(w => /[a-zA-Z]/.test(w)).length; const minRu = args.minRu || 5; const minEn = args.minEn || 5; if (ruWords < minRu) { return { pass: false, reason: `Мало русских слов: ${ruWords} (надо минимум ${minRu})` }; } if (enWords < minEn) { return { pass: false, reason: `Мало английских слов: ${enWords} (надо минимум ${minEn})` }; } return { pass: true, reason: `Code-switching OK (${ruWords} RU / ${enWords} EN слов)` }; } case "include_keywords": { const missing = (args.keywords || []).filter(kw => !lower.includes(kw.toLowerCase())); return { pass: missing.length === 0, reason: missing.length === 0 ? "Все ключевые слова найдены" : `Нет слов: ${missing.join(", ")}` }; } case "forbidden_words": { const found = (args.words || []).filter(w => lower.includes(w.toLowerCase())); return { pass: found.length === 0, reason: found.length === 0 ? "Запрещенных слов нет" : `Найдены запрещенные слова: ${found.join(", ")}` }; } case "keyword_frequency": { const escaped = args.word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const re = new RegExp(`(? lower.includes(s)); return { pass, reason: pass ? `Язык (${args.lang}) подтвержден` : `Маркеры (${args.lang}) не найдены` }; } case "number_words": { const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w)); const count = words.length; if (args.exact) return { pass: count === args.exact, reason: `Слов: ${count} (надо ровно ${args.exact})` }; if (args.min && count < args.min) return { pass: false, reason: `Слов: ${count} (мин. ${args.min})` }; if (args.max && count > args.max) return { pass: false, reason: `Слов: ${count} (макс. ${args.max})` }; return { pass: true, reason: `Слов в норме (${count})` }; } case "number_sentences": { const normalized = clean.replace(/P\.S\./gi, "PS"); const count = normalized.split(/[.!?]+/).filter(s => s.trim().length > 0).length; return { pass: count === args.count, reason: `Предложений: ${count} (надо ровно ${args.count})` }; } case "number_paragraphs": { const count = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0).length; return { pass: count === args.count, reason: `Абзацев: ${count} (надо ровно ${args.count})` }; } case "paragraph_first_word": { const paras = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0); const target = paras[args.paragraphIndex]; if (!target) return { pass: false, reason: `Абзац ${args.paragraphIndex + 1} отсутствует` }; const firstWord = target.split(/\s+/)[0].replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}]+$/u, ""); const pass = firstWord.toLowerCase() === args.word.toLowerCase(); return { pass, reason: pass ? `Первое слово: '${firstWord}'` : `Первое слово: '${firstWord}' (ожидалось '${args.word}')` }; } case "number_bullet_lists": { const bullets = clean.split("\n").filter(line => /^\s*([-*•]|\d+[.)])\s+/.test(line)); return { pass: bullets.length === args.count, reason: `Пунктов списка: ${bullets.length} (надо ${args.count})` }; } case "title": { const pass = /<<[^>]+>>/.test(clean); return { pass, reason: pass ? "Заголовок <<...>> найден" : "Заголовок <<...>> не найден" }; } case "constrained_response": { const pass = clean.toLowerCase() === (args.expected || "").toLowerCase(); return { pass, reason: pass ? "Точный ответ" : `Получено '${clean}' (ожидалось '${args.expected}')` }; } case "highlighted_section": { const matches = clean.match(/(?= min, reason: `Выделений звездочками: ${count} (мин. ${min})` }; } case "multiple_sections": { const matches = clean.match(/(?:^|\n)\s*(?:[#*_\->•]|\d+[.)]|\s)*\bSection\s+\d+\b/gi) || []; return { pass: matches.length === args.count, reason: `Секций Section X: ${matches.length} (надо ${args.count})` }; } case "json_format": { try { const c = clean.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/, "").trim(); const parsed = JSON.parse(c); if (args.isArray && !Array.isArray(parsed)) return { pass: false, reason: "Не массив JSON" }; if (Array.isArray(args.requiredKeys)) { const obj = Array.isArray(parsed) ? parsed[0] : parsed; if (!obj || typeof obj !== "object") return { pass: false, reason: "Некорректный объект JSON" }; for (const k of args.requiredKeys) { if (!(k in obj)) return { pass: false, reason: `Нет ключа '${k}' в JSON` }; } } return { pass: true, reason: "Валидный JSON с нужными ключами" }; } catch { return { pass: false, reason: "Не является валидным JSON" }; } } case "number_of_placeholders": { const matches = clean.match(/\[[^\]\n]+\]/g) || []; const count = matches.length; if (args.exact) return { pass: count === args.exact, reason: `Плейсхолдеров [...]: ${count} (надо ${args.exact})` }; const min = args.min || 1; return { pass: count >= min, reason: `Плейсхолдеров [...]: ${count} (мин. ${min})` }; } case "postscript": { const pass = /(?:\n|^)\s*P\.S\..*$/is.test(clean); return { pass, reason: pass ? "P.S. найден в конце" : "P.S. в конце отсутствует" }; } case "repeat_prompt": { const promptClean = (args.prompt || "").trim().toLowerCase(); const pass = lower.startsWith(promptClean); return { pass, reason: pass ? "Запрос повторен дословно" : "Запрос не повторен дословно в начале" }; } case "two_responses": { const parts = clean.split("******"); const pass = parts.length === 2 && parts[0].trim().length > 0 && parts[1].trim().length > 0; return { pass, reason: pass ? "Два ответа через ******" : "Разделитель ****** отсутствует" }; } case "all_uppercase": { const hasL = /\p{L}/u.test(clean); const pass = hasL && clean === clean.toUpperCase(); return { pass, reason: pass ? "Все ЗАГЛАВНЫМИ БУКВАМИ" : "Найдены строчные буквы" }; } case "all_lowercase": { const hasL = /\p{L}/u.test(clean); const pass = hasL && clean === clean.toLowerCase(); return { pass, reason: pass ? "Все строчными буквами" : "Найдены заглавные буквы" }; } case "capital_word_frequency": { const rawWords = clean.split(/\s+/).filter(Boolean); const capWords = []; for (let i = 0; i < rawWords.length; i++) { const w = rawWords[i]; const letters = w.replace(/[^\p{L}]/gu, ""); if (!letters || letters !== letters.toUpperCase()) continue; if (letters.length >= 2) capWords.push(letters); else if (letters.length === 1) { const prev = i > 0 ? rawWords[i - 1] : ""; const isStart = i === 0 || /[.!?][)"'»*]*$/.test(prev); const next = i < rawWords.length - 1 ? rawWords[i + 1] : ""; if (isStart && /\p{Ll}/u.test(next)) continue; capWords.push(letters); } } return { pass: capWords.length === args.count, reason: `Слов капсом: ${capWords.length} (надо ${args.count})` }; } case "end_checker": { const pass = clean.endsWith(args.phrase); return { pass, reason: pass ? `Фраза в конце: '${args.phrase}'` : `Нет фразы '${args.phrase}' в конце` }; } case "quotation": { const pass = (clean.startsWith('"') && clean.endsWith('"')) || (clean.startsWith("«") && clean.endsWith("»")); return { pass, reason: pass ? "Весь текст в кавычках" : "Текст не заключен в кавычки целиком" }; } case "no_comma": { const pass = !clean.includes(",") && !clean.includes(","); return { pass, reason: pass ? "Без запятых" : "Найдена запрещенная запятая" }; } default: return { pass: true, reason: "Неизвестный тип валидатора" }; } } function validateResponse(scenario, rawText) { const validators = scenario.validators || (scenario.validatorType ? [{ type: scenario.validatorType, args: scenario.validatorArgs }] : []); if (validators.length === 0) return { pass: true, reason: "Нет валидаторов" }; const results = validators.map(v => { const vArgs = { ...(v.args || {}) }; if (v.type === "repeat_prompt") { vArgs.prompt = scenario.prompt; } return { type: v.type, res: validateSingleRule(v.type, vArgs, rawText) }; }); const allPass = results.every(r => r.res.pass); const summary = results.map(r => `[${r.res.pass ? "OK" : "FAIL"}: ${r.type} -> ${r.res.reason}]`).join(" "); return { pass: allPass, reason: summary }; } function resolveHelpers(context) { if (context && typeof context.getRequiredProvider === "function") { return context; } if (context?.helpers && typeof context.helpers.getRequiredProvider === "function") { return context.helpers; } return { getRequiredProvider(providerId) { const providers = context?.providers || []; const found = Array.isArray(providers) ? providers.find((p) => p.id === providerId || p.provider === providerId) : providers[providerId]; return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" }; }, getInferenceEndpoint(modelId) { return context?.endpoints?.[modelId] || null; } }; } export async function prepare(context) { const helpers = resolveHelpers(context); return { async runScenario(input) { const allScenarios = loadScenarios(); const scenario = allScenarios.find(s => s.id === input.scenario.id); if (!scenario) { return { scenarioId: input.scenario.id, status: "fail", score: 0, summary: "Сценарий не найден", rawLog: "error=Scenario not found" }; } const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true }); const inference = helpers.getInferenceEndpoint ? helpers.getInferenceEndpoint(input.model.id) : null; const rawBaseUrl = (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) || "http://127.0.0.1:11434/v1"; const baseUrl = rawBaseUrl.replace(/\/+$/, ""); const apiKey = provider?.apiKey || "none"; const requestBody = { model: input.model.model, messages: [{ role: "user", content: scenario.prompt }], temperature: 0 }; const response = await fetch(`${baseUrl}/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, body: JSON.stringify(requestBody) }); const data = await response.json(); const text = (data.choices?.[0]?.message?.content || "").trim(); const result = validateResponse(scenario, text); const rawLog = [ `model=${input.model.model}`, `scenario=${scenario.id} ${scenario.name}`, `user_turn_1=${scenario.prompt}`, `assistant_turn_1=${text}`, `final_answer=${text}`, `verdict=${result.pass ? "pass" : "fail"}`, `score=${result.pass ? 100 : 0}`, `summary=${result.reason}`, `\n--- RAW API RESPONSE ---`, JSON.stringify(data, null, 2) ].join("\n"); return { scenarioId: scenario.id, status: result.pass ? "pass" : "fail", score: result.pass ? 100 : 0, summary: result.reason, rawLog: rawLog }; }, async dispose() {} }; } export function scoreModelResults(results) { const scored = results || []; const total = scored.reduce((sum, r) => sum + (r.score || 0), 0); const max = scored.length * 100; return { totalScore: max > 0 ? Math.round((total / max) * 100) : 0, categories: [] }; } export default { manifest, listScenarios, getScenarios, prepare, createRuntime: prepare, scoreModelResults };