Download ifeval-mini/index.mjs from zelk12/BenchPack_for_BenchLocal: direct link, hf CLI and curl.
- Browser
- Download file 16.8 kB
-
https://huggingface.co/zelk12/BenchPack_for_BenchLocal/resolve/main/ifeval-mini/index.mjs
- Command line
-
hf download hf://zelk12/BenchPack_for_BenchLocal/ifeval-mini/index.mjs
-
curl -L -o index.mjs https://huggingface.co/zelk12/BenchPack_for_BenchLocal/resolve/main/ifeval-mini/index.mjs
16.8 kB
| import fs from "node:fs"; | |
| import path from "node:path"; | |
| import { fileURLToPath } from "node:url"; | |
| const __dirname = path.dirname(fileURLToPath(import.meta.url)); | |
| const manifestPath = path.join(__dirname, "benchlocal.pack.json"); | |
| const scenariosPath = path.join(__dirname, "scenarios.json"); | |
| function readJsonSafe(filePath, fallback) { | |
| try { | |
| if (fs.existsSync(filePath)) { | |
| const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "").trim(); | |
| return JSON.parse(raw); | |
| } | |
| } catch (err) { | |
| console.error(`[IFEval-Hard] Ошибка чтения ${filePath}:`, err.message); | |
| } | |
| return fallback; | |
| } | |
| const defaultManifest = { | |
| schemaVersion: 1, | |
| protocolVersion: 1, | |
| id: "ifeval-hard", | |
| name: "IFEval-Hard (90 комплексных задач)", | |
| version: "1.0.0", | |
| description: "Аутентичный сложный IFEval с мульти-композициями: 30 RU, 30 EN и 30 Code-Switching задач (по 2-3 строгих ограничения на промпт)", | |
| entry: "./index.mjs", | |
| samplingDefaults: { temperature: 0.0 }, | |
| capabilities: { tools: false, multiTurn: false, streamingProgress: false, verification: false } | |
| }; | |
| export const manifest = readJsonSafe(manifestPath, defaultManifest); | |
| function loadScenarios() { | |
| return readJsonSafe(scenariosPath, []); | |
| } | |
| export function listScenarios() { | |
| return loadScenarios().map(s => ({ | |
| id: s.id, | |
| name: s.name, | |
| category: s.category || "Общие", | |
| description: s.name | |
| })); | |
| } | |
| export function getScenarios() { | |
| return listScenarios(); | |
| } | |
| function validateSingleRule(type, args, text) { | |
| const clean = (text || "").trim(); | |
| const lower = clean.toLowerCase(); | |
| switch (type) { | |
| case "code_switching": { | |
| const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w)); | |
| const ruWords = words.filter(w => /[а-яА-ЯёЁ]/.test(w)).length; | |
| const enWords = words.filter(w => /[a-zA-Z]/.test(w)).length; | |
| const minRu = args.minRu || 5; | |
| const minEn = args.minEn || 5; | |
| if (ruWords < minRu) { | |
| return { pass: false, reason: `Мало русских слов: ${ruWords} (надо минимум ${minRu})` }; | |
| } | |
| if (enWords < minEn) { | |
| return { pass: false, reason: `Мало английских слов: ${enWords} (надо минимум ${minEn})` }; | |
| } | |
| return { pass: true, reason: `Code-switching OK (${ruWords} RU / ${enWords} EN слов)` }; | |
| } | |
| case "include_keywords": { | |
| const missing = (args.keywords || []).filter(kw => !lower.includes(kw.toLowerCase())); | |
| return { | |
| pass: missing.length === 0, | |
| reason: missing.length === 0 ? "Все ключевые слова найдены" : `Нет слов: ${missing.join(", ")}` | |
| }; | |
| } | |
| case "forbidden_words": { | |
| const found = (args.words || []).filter(w => lower.includes(w.toLowerCase())); | |
| return { | |
| pass: found.length === 0, | |
| reason: found.length === 0 ? "Запрещенных слов нет" : `Найдены запрещенные слова: ${found.join(", ")}` | |
| }; | |
| } | |
| case "keyword_frequency": { | |
| const escaped = args.word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | |
| const re = new RegExp(`(?<!\\p{L})${escaped}(?!\\p{L})`, "gui"); | |
| const count = (lower.match(re) || []).length; | |
| return { | |
| pass: count === args.count, | |
| reason: count === args.count ? `Слово '${args.word}' x${count}` : `Слово '${args.word}' x${count} (требовалось ${args.count})` | |
| }; | |
| } | |
| case "letter_frequency": { | |
| const char = args.letter.toLowerCase(); | |
| const count = lower.split(char).length - 1; | |
| return { | |
| pass: count === args.count, | |
| reason: count === args.count ? `Буква '${char}' x${count}` : `Буква '${char}' x${count} (требовалось ${args.count})` | |
| }; | |
| } | |
| case "response_language": { | |
| if (args.lang === "ru") { | |
| const pass = !/[a-zA-Z]/.test(clean) && /[а-яА-ЯёЁ]/.test(clean); | |
| return { pass, reason: pass ? "Строго русский язык без латиницы" : "Обнаружена латиница" }; | |
| } | |
| const samples = { | |
| es: ["de", "la", "el", "en"], | |
| fr: ["le", "la", "du", "est"], | |
| de: ["und", "der", "die", "das"], | |
| it: ["di", "per", "con", "il"] | |
| }[args.lang] || []; | |
| const pass = samples.some(s => lower.includes(s)); | |
| return { pass, reason: pass ? `Язык (${args.lang}) подтвержден` : `Маркеры (${args.lang}) не найдены` }; | |
| } | |
| case "number_words": { | |
| const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w)); | |
| const count = words.length; | |
| if (args.exact) return { pass: count === args.exact, reason: `Слов: ${count} (надо ровно ${args.exact})` }; | |
| if (args.min && count < args.min) return { pass: false, reason: `Слов: ${count} (мин. ${args.min})` }; | |
| if (args.max && count > args.max) return { pass: false, reason: `Слов: ${count} (макс. ${args.max})` }; | |
| return { pass: true, reason: `Слов в норме (${count})` }; | |
| } | |
| case "number_sentences": { | |
| const normalized = clean.replace(/P\.S\./gi, "PS"); | |
| const count = normalized.split(/[.!?]+/).filter(s => s.trim().length > 0).length; | |
| return { pass: count === args.count, reason: `Предложений: ${count} (надо ровно ${args.count})` }; | |
| } | |
| case "number_paragraphs": { | |
| const count = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0).length; | |
| return { pass: count === args.count, reason: `Абзацев: ${count} (надо ровно ${args.count})` }; | |
| } | |
| case "paragraph_first_word": { | |
| const paras = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0); | |
| const target = paras[args.paragraphIndex]; | |
| if (!target) return { pass: false, reason: `Абзац ${args.paragraphIndex + 1} отсутствует` }; | |
| const firstWord = target.split(/\s+/)[0].replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}]+$/u, ""); | |
| const pass = firstWord.toLowerCase() === args.word.toLowerCase(); | |
| return { pass, reason: pass ? `Первое слово: '${firstWord}'` : `Первое слово: '${firstWord}' (ожидалось '${args.word}')` }; | |
| } | |
| case "number_bullet_lists": { | |
| const bullets = clean.split("\n").filter(line => /^\s*([-*•]|\d+[.)])\s+/.test(line)); | |
| return { pass: bullets.length === args.count, reason: `Пунктов списка: ${bullets.length} (надо ${args.count})` }; | |
| } | |
| case "title": { | |
| const pass = /<<[^>]+>>/.test(clean); | |
| return { pass, reason: pass ? "Заголовок <<...>> найден" : "Заголовок <<...>> не найден" }; | |
| } | |
| case "constrained_response": { | |
| const pass = clean.toLowerCase() === (args.expected || "").toLowerCase(); | |
| return { pass, reason: pass ? "Точный ответ" : `Получено '${clean}' (ожидалось '${args.expected}')` }; | |
| } | |
| case "highlighted_section": { | |
| const matches = clean.match(/(?<!\*)\*(?!\*)([^*\n]+?)(?<!\*)\*(?!\*)/g) || []; | |
| const count = matches.length; | |
| if (args.exact) return { pass: count === args.exact, reason: `Выделений звездочками: ${count} (надо ${args.exact})` }; | |
| const min = args.min || 1; | |
| return { pass: count >= min, reason: `Выделений звездочками: ${count} (мин. ${min})` }; | |
| } | |
| case "multiple_sections": { | |
| const matches = clean.match(/(?:^|\n)\s*(?:[#*_\->•]|\d+[.)]|\s)*\bSection\s+\d+\b/gi) || []; | |
| return { pass: matches.length === args.count, reason: `Секций Section X: ${matches.length} (надо ${args.count})` }; | |
| } | |
| case "json_format": { | |
| try { | |
| const c = clean.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/, "").trim(); | |
| const parsed = JSON.parse(c); | |
| if (args.isArray && !Array.isArray(parsed)) return { pass: false, reason: "Не массив JSON" }; | |
| if (Array.isArray(args.requiredKeys)) { | |
| const obj = Array.isArray(parsed) ? parsed[0] : parsed; | |
| if (!obj || typeof obj !== "object") return { pass: false, reason: "Некорректный объект JSON" }; | |
| for (const k of args.requiredKeys) { | |
| if (!(k in obj)) return { pass: false, reason: `Нет ключа '${k}' в JSON` }; | |
| } | |
| } | |
| return { pass: true, reason: "Валидный JSON с нужными ключами" }; | |
| } catch { | |
| return { pass: false, reason: "Не является валидным JSON" }; | |
| } | |
| } | |
| case "number_of_placeholders": { | |
| const matches = clean.match(/\[[^\]\n]+\]/g) || []; | |
| const count = matches.length; | |
| if (args.exact) return { pass: count === args.exact, reason: `Плейсхолдеров [...]: ${count} (надо ${args.exact})` }; | |
| const min = args.min || 1; | |
| return { pass: count >= min, reason: `Плейсхолдеров [...]: ${count} (мин. ${min})` }; | |
| } | |
| case "postscript": { | |
| const pass = /(?:\n|^)\s*P\.S\..*$/is.test(clean); | |
| return { pass, reason: pass ? "P.S. найден в конце" : "P.S. в конце отсутствует" }; | |
| } | |
| case "repeat_prompt": { | |
| const promptClean = (args.prompt || "").trim().toLowerCase(); | |
| const pass = lower.startsWith(promptClean); | |
| return { pass, reason: pass ? "Запрос повторен дословно" : "Запрос не повторен дословно в начале" }; | |
| } | |
| case "two_responses": { | |
| const parts = clean.split("******"); | |
| const pass = parts.length === 2 && parts[0].trim().length > 0 && parts[1].trim().length > 0; | |
| return { pass, reason: pass ? "Два ответа через ******" : "Разделитель ****** отсутствует" }; | |
| } | |
| case "all_uppercase": { | |
| const hasL = /\p{L}/u.test(clean); | |
| const pass = hasL && clean === clean.toUpperCase(); | |
| return { pass, reason: pass ? "Все ЗАГЛАВНЫМИ БУКВАМИ" : "Найдены строчные буквы" }; | |
| } | |
| case "all_lowercase": { | |
| const hasL = /\p{L}/u.test(clean); | |
| const pass = hasL && clean === clean.toLowerCase(); | |
| return { pass, reason: pass ? "Все строчными буквами" : "Найдены заглавные буквы" }; | |
| } | |
| case "capital_word_frequency": { | |
| const rawWords = clean.split(/\s+/).filter(Boolean); | |
| const capWords = []; | |
| for (let i = 0; i < rawWords.length; i++) { | |
| const w = rawWords[i]; | |
| const letters = w.replace(/[^\p{L}]/gu, ""); | |
| if (!letters || letters !== letters.toUpperCase()) continue; | |
| if (letters.length >= 2) capWords.push(letters); | |
| else if (letters.length === 1) { | |
| const prev = i > 0 ? rawWords[i - 1] : ""; | |
| const isStart = i === 0 || /[.!?][)"'»*]*$/.test(prev); | |
| const next = i < rawWords.length - 1 ? rawWords[i + 1] : ""; | |
| if (isStart && /\p{Ll}/u.test(next)) continue; | |
| capWords.push(letters); | |
| } | |
| } | |
| return { pass: capWords.length === args.count, reason: `Слов капсом: ${capWords.length} (надо ${args.count})` }; | |
| } | |
| case "end_checker": { | |
| const pass = clean.endsWith(args.phrase); | |
| return { pass, reason: pass ? `Фраза в конце: '${args.phrase}'` : `Нет фразы '${args.phrase}' в конце` }; | |
| } | |
| case "quotation": { | |
| const pass = (clean.startsWith('"') && clean.endsWith('"')) || (clean.startsWith("«") && clean.endsWith("»")); | |
| return { pass, reason: pass ? "Весь текст в кавычках" : "Текст не заключен в кавычки целиком" }; | |
| } | |
| case "no_comma": { | |
| const pass = !clean.includes(",") && !clean.includes(","); | |
| return { pass, reason: pass ? "Без запятых" : "Найдена запрещенная запятая" }; | |
| } | |
| default: | |
| return { pass: true, reason: "Неизвестный тип валидатора" }; | |
| } | |
| } | |
| function validateResponse(scenario, rawText) { | |
| const validators = scenario.validators || (scenario.validatorType ? [{ type: scenario.validatorType, args: scenario.validatorArgs }] : []); | |
| if (validators.length === 0) return { pass: true, reason: "Нет валидаторов" }; | |
| const results = validators.map(v => { | |
| const vArgs = { ...(v.args || {}) }; | |
| if (v.type === "repeat_prompt") { | |
| vArgs.prompt = scenario.prompt; | |
| } | |
| return { | |
| type: v.type, | |
| res: validateSingleRule(v.type, vArgs, rawText) | |
| }; | |
| }); | |
| const allPass = results.every(r => r.res.pass); | |
| const summary = results.map(r => `[${r.res.pass ? "OK" : "FAIL"}: ${r.type} -> ${r.res.reason}]`).join(" "); | |
| return { | |
| pass: allPass, | |
| reason: summary | |
| }; | |
| } | |
| function resolveHelpers(context) { | |
| if (context && typeof context.getRequiredProvider === "function") { | |
| return context; | |
| } | |
| if (context?.helpers && typeof context.helpers.getRequiredProvider === "function") { | |
| return context.helpers; | |
| } | |
| return { | |
| getRequiredProvider(providerId) { | |
| const providers = context?.providers || []; | |
| const found = Array.isArray(providers) | |
| ? providers.find((p) => p.id === providerId || p.provider === providerId) | |
| : providers[providerId]; | |
| return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" }; | |
| }, | |
| getInferenceEndpoint(modelId) { | |
| return context?.endpoints?.[modelId] || null; | |
| } | |
| }; | |
| } | |
| export async function prepare(context) { | |
| const helpers = resolveHelpers(context); | |
| return { | |
| async runScenario(input) { | |
| const allScenarios = loadScenarios(); | |
| const scenario = allScenarios.find(s => s.id === input.scenario.id); | |
| if (!scenario) { | |
| return { | |
| scenarioId: input.scenario.id, | |
| status: "fail", | |
| score: 0, | |
| summary: "Сценарий не найден", | |
| rawLog: "error=Scenario not found" | |
| }; | |
| } | |
| const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true }); | |
| const inference = helpers.getInferenceEndpoint ? helpers.getInferenceEndpoint(input.model.id) : null; | |
| const rawBaseUrl = (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) || "http://127.0.0.1:11434/v1"; | |
| const baseUrl = rawBaseUrl.replace(/\/+$/, ""); | |
| const apiKey = provider?.apiKey || "none"; | |
| const requestBody = { | |
| model: input.model.model, | |
| messages: [{ role: "user", content: scenario.prompt }], | |
| temperature: 0 | |
| }; | |
| const response = await fetch(`${baseUrl}/chat/completions`, { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| Authorization: `Bearer ${apiKey}` | |
| }, | |
| body: JSON.stringify(requestBody) | |
| }); | |
| const data = await response.json(); | |
| const text = (data.choices?.[0]?.message?.content || "").trim(); | |
| const result = validateResponse(scenario, text); | |
| const rawLog = [ | |
| `model=${input.model.model}`, | |
| `scenario=${scenario.id} ${scenario.name}`, | |
| `user_turn_1=${scenario.prompt}`, | |
| `assistant_turn_1=${text}`, | |
| `final_answer=${text}`, | |
| `verdict=${result.pass ? "pass" : "fail"}`, | |
| `score=${result.pass ? 100 : 0}`, | |
| `summary=${result.reason}`, | |
| `\n--- RAW API RESPONSE ---`, | |
| JSON.stringify(data, null, 2) | |
| ].join("\n"); | |
| return { | |
| scenarioId: scenario.id, | |
| status: result.pass ? "pass" : "fail", | |
| score: result.pass ? 100 : 0, | |
| summary: result.reason, | |
| rawLog: rawLog | |
| }; | |
| }, | |
| async dispose() {} | |
| }; | |
| } | |
| export function scoreModelResults(results) { | |
| const scored = results || []; | |
| const total = scored.reduce((sum, r) => sum + (r.score || 0), 0); | |
| const max = scored.length * 100; | |
| return { | |
| totalScore: max > 0 ? Math.round((total / max) * 100) : 0, | |
| categories: [] | |
| }; | |
| } | |
| export default { | |
| manifest, | |
| listScenarios, | |
| getScenarios, | |
| prepare, | |
| createRuntime: prepare, | |
| scoreModelResults | |
| }; |