zelk12's picture
Upload 3 files
27111ee verified
Raw History Blame Contribute Delete
16.8 kB
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const manifestPath = path.join(__dirname, "benchlocal.pack.json");
const scenariosPath = path.join(__dirname, "scenarios.json");
function readJsonSafe(filePath, fallback) {
try {
if (fs.existsSync(filePath)) {
const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "").trim();
return JSON.parse(raw);
}
} catch (err) {
console.error(`[IFEval-Hard] Ошибка чтения ${filePath}:`, err.message);
}
return fallback;
}
const defaultManifest = {
schemaVersion: 1,
protocolVersion: 1,
id: "ifeval-hard",
name: "IFEval-Hard (90 комплексных задач)",
version: "1.0.0",
description: "Аутентичный сложный IFEval с мульти-композициями: 30 RU, 30 EN и 30 Code-Switching задач (по 2-3 строгих ограничения на промпт)",
entry: "./index.mjs",
samplingDefaults: { temperature: 0.0 },
capabilities: { tools: false, multiTurn: false, streamingProgress: false, verification: false }
};
export const manifest = readJsonSafe(manifestPath, defaultManifest);
function loadScenarios() {
return readJsonSafe(scenariosPath, []);
}
export function listScenarios() {
return loadScenarios().map(s => ({
id: s.id,
name: s.name,
category: s.category || "Общие",
description: s.name
}));
}
export function getScenarios() {
return listScenarios();
}
function validateSingleRule(type, args, text) {
const clean = (text || "").trim();
const lower = clean.toLowerCase();
switch (type) {
case "code_switching": {
const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w));
const ruWords = words.filter(w => /[а-яА-ЯёЁ]/.test(w)).length;
const enWords = words.filter(w => /[a-zA-Z]/.test(w)).length;
const minRu = args.minRu || 5;
const minEn = args.minEn || 5;
if (ruWords < minRu) {
return { pass: false, reason: `Мало русских слов: ${ruWords} (надо минимум ${minRu})` };
}
if (enWords < minEn) {
return { pass: false, reason: `Мало английских слов: ${enWords} (надо минимум ${minEn})` };
}
return { pass: true, reason: `Code-switching OK (${ruWords} RU / ${enWords} EN слов)` };
}
case "include_keywords": {
const missing = (args.keywords || []).filter(kw => !lower.includes(kw.toLowerCase()));
return {
pass: missing.length === 0,
reason: missing.length === 0 ? "Все ключевые слова найдены" : `Нет слов: ${missing.join(", ")}`
};
}
case "forbidden_words": {
const found = (args.words || []).filter(w => lower.includes(w.toLowerCase()));
return {
pass: found.length === 0,
reason: found.length === 0 ? "Запрещенных слов нет" : `Найдены запрещенные слова: ${found.join(", ")}`
};
}
case "keyword_frequency": {
const escaped = args.word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const re = new RegExp(`(?<!\\p{L})${escaped}(?!\\p{L})`, "gui");
const count = (lower.match(re) || []).length;
return {
pass: count === args.count,
reason: count === args.count ? `Слово '${args.word}' x${count}` : `Слово '${args.word}' x${count} (требовалось ${args.count})`
};
}
case "letter_frequency": {
const char = args.letter.toLowerCase();
const count = lower.split(char).length - 1;
return {
pass: count === args.count,
reason: count === args.count ? `Буква '${char}' x${count}` : `Буква '${char}' x${count} (требовалось ${args.count})`
};
}
case "response_language": {
if (args.lang === "ru") {
const pass = !/[a-zA-Z]/.test(clean) && /[а-яА-ЯёЁ]/.test(clean);
return { pass, reason: pass ? "Строго русский язык без латиницы" : "Обнаружена латиница" };
}
const samples = {
es: ["de", "la", "el", "en"],
fr: ["le", "la", "du", "est"],
de: ["und", "der", "die", "das"],
it: ["di", "per", "con", "il"]
}[args.lang] || [];
const pass = samples.some(s => lower.includes(s));
return { pass, reason: pass ? `Язык (${args.lang}) подтвержден` : `Маркеры (${args.lang}) не найдены` };
}
case "number_words": {
const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w));
const count = words.length;
if (args.exact) return { pass: count === args.exact, reason: `Слов: ${count} (надо ровно ${args.exact})` };
if (args.min && count < args.min) return { pass: false, reason: `Слов: ${count} (мин. ${args.min})` };
if (args.max && count > args.max) return { pass: false, reason: `Слов: ${count} (макс. ${args.max})` };
return { pass: true, reason: `Слов в норме (${count})` };
}
case "number_sentences": {
const normalized = clean.replace(/P\.S\./gi, "PS");
const count = normalized.split(/[.!?]+/).filter(s => s.trim().length > 0).length;
return { pass: count === args.count, reason: `Предложений: ${count} (надо ровно ${args.count})` };
}
case "number_paragraphs": {
const count = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0).length;
return { pass: count === args.count, reason: `Абзацев: ${count} (надо ровно ${args.count})` };
}
case "paragraph_first_word": {
const paras = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0);
const target = paras[args.paragraphIndex];
if (!target) return { pass: false, reason: `Абзац ${args.paragraphIndex + 1} отсутствует` };
const firstWord = target.split(/\s+/)[0].replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}]+$/u, "");
const pass = firstWord.toLowerCase() === args.word.toLowerCase();
return { pass, reason: pass ? `Первое слово: '${firstWord}'` : `Первое слово: '${firstWord}' (ожидалось '${args.word}')` };
}
case "number_bullet_lists": {
const bullets = clean.split("\n").filter(line => /^\s*([-*•]|\d+[.)])\s+/.test(line));
return { pass: bullets.length === args.count, reason: `Пунктов списка: ${bullets.length} (надо ${args.count})` };
}
case "title": {
const pass = /<<[^>]+>>/.test(clean);
return { pass, reason: pass ? "Заголовок <<...>> найден" : "Заголовок <<...>> не найден" };
}
case "constrained_response": {
const pass = clean.toLowerCase() === (args.expected || "").toLowerCase();
return { pass, reason: pass ? "Точный ответ" : `Получено '${clean}' (ожидалось '${args.expected}')` };
}
case "highlighted_section": {
const matches = clean.match(/(?<!\*)\*(?!\*)([^*\n]+?)(?<!\*)\*(?!\*)/g) || [];
const count = matches.length;
if (args.exact) return { pass: count === args.exact, reason: `Выделений звездочками: ${count} (надо ${args.exact})` };
const min = args.min || 1;
return { pass: count >= min, reason: `Выделений звездочками: ${count} (мин. ${min})` };
}
case "multiple_sections": {
const matches = clean.match(/(?:^|\n)\s*(?:[#*_\->•]|\d+[.)]|\s)*\bSection\s+\d+\b/gi) || [];
return { pass: matches.length === args.count, reason: `Секций Section X: ${matches.length} (надо ${args.count})` };
}
case "json_format": {
try {
const c = clean.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/, "").trim();
const parsed = JSON.parse(c);
if (args.isArray && !Array.isArray(parsed)) return { pass: false, reason: "Не массив JSON" };
if (Array.isArray(args.requiredKeys)) {
const obj = Array.isArray(parsed) ? parsed[0] : parsed;
if (!obj || typeof obj !== "object") return { pass: false, reason: "Некорректный объект JSON" };
for (const k of args.requiredKeys) {
if (!(k in obj)) return { pass: false, reason: `Нет ключа '${k}' в JSON` };
}
}
return { pass: true, reason: "Валидный JSON с нужными ключами" };
} catch {
return { pass: false, reason: "Не является валидным JSON" };
}
}
case "number_of_placeholders": {
const matches = clean.match(/\[[^\]\n]+\]/g) || [];
const count = matches.length;
if (args.exact) return { pass: count === args.exact, reason: `Плейсхолдеров [...]: ${count} (надо ${args.exact})` };
const min = args.min || 1;
return { pass: count >= min, reason: `Плейсхолдеров [...]: ${count} (мин. ${min})` };
}
case "postscript": {
const pass = /(?:\n|^)\s*P\.S\..*$/is.test(clean);
return { pass, reason: pass ? "P.S. найден в конце" : "P.S. в конце отсутствует" };
}
case "repeat_prompt": {
const promptClean = (args.prompt || "").trim().toLowerCase();
const pass = lower.startsWith(promptClean);
return { pass, reason: pass ? "Запрос повторен дословно" : "Запрос не повторен дословно в начале" };
}
case "two_responses": {
const parts = clean.split("******");
const pass = parts.length === 2 && parts[0].trim().length > 0 && parts[1].trim().length > 0;
return { pass, reason: pass ? "Два ответа через ******" : "Разделитель ****** отсутствует" };
}
case "all_uppercase": {
const hasL = /\p{L}/u.test(clean);
const pass = hasL && clean === clean.toUpperCase();
return { pass, reason: pass ? "Все ЗАГЛАВНЫМИ БУКВАМИ" : "Найдены строчные буквы" };
}
case "all_lowercase": {
const hasL = /\p{L}/u.test(clean);
const pass = hasL && clean === clean.toLowerCase();
return { pass, reason: pass ? "Все строчными буквами" : "Найдены заглавные буквы" };
}
case "capital_word_frequency": {
const rawWords = clean.split(/\s+/).filter(Boolean);
const capWords = [];
for (let i = 0; i < rawWords.length; i++) {
const w = rawWords[i];
const letters = w.replace(/[^\p{L}]/gu, "");
if (!letters || letters !== letters.toUpperCase()) continue;
if (letters.length >= 2) capWords.push(letters);
else if (letters.length === 1) {
const prev = i > 0 ? rawWords[i - 1] : "";
const isStart = i === 0 || /[.!?][)"'»*]*$/.test(prev);
const next = i < rawWords.length - 1 ? rawWords[i + 1] : "";
if (isStart && /\p{Ll}/u.test(next)) continue;
capWords.push(letters);
}
}
return { pass: capWords.length === args.count, reason: `Слов капсом: ${capWords.length} (надо ${args.count})` };
}
case "end_checker": {
const pass = clean.endsWith(args.phrase);
return { pass, reason: pass ? `Фраза в конце: '${args.phrase}'` : `Нет фразы '${args.phrase}' в конце` };
}
case "quotation": {
const pass = (clean.startsWith('"') && clean.endsWith('"')) || (clean.startsWith("«") && clean.endsWith("»"));
return { pass, reason: pass ? "Весь текст в кавычках" : "Текст не заключен в кавычки целиком" };
}
case "no_comma": {
const pass = !clean.includes(",") && !clean.includes(",");
return { pass, reason: pass ? "Без запятых" : "Найдена запрещенная запятая" };
}
default:
return { pass: true, reason: "Неизвестный тип валидатора" };
}
}
function validateResponse(scenario, rawText) {
const validators = scenario.validators || (scenario.validatorType ? [{ type: scenario.validatorType, args: scenario.validatorArgs }] : []);
if (validators.length === 0) return { pass: true, reason: "Нет валидаторов" };
const results = validators.map(v => {
const vArgs = { ...(v.args || {}) };
if (v.type === "repeat_prompt") {
vArgs.prompt = scenario.prompt;
}
return {
type: v.type,
res: validateSingleRule(v.type, vArgs, rawText)
};
});
const allPass = results.every(r => r.res.pass);
const summary = results.map(r => `[${r.res.pass ? "OK" : "FAIL"}: ${r.type} -> ${r.res.reason}]`).join(" ");
return {
pass: allPass,
reason: summary
};
}
function resolveHelpers(context) {
if (context && typeof context.getRequiredProvider === "function") {
return context;
}
if (context?.helpers && typeof context.helpers.getRequiredProvider === "function") {
return context.helpers;
}
return {
getRequiredProvider(providerId) {
const providers = context?.providers || [];
const found = Array.isArray(providers)
? providers.find((p) => p.id === providerId || p.provider === providerId)
: providers[providerId];
return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" };
},
getInferenceEndpoint(modelId) {
return context?.endpoints?.[modelId] || null;
}
};
}
export async function prepare(context) {
const helpers = resolveHelpers(context);
return {
async runScenario(input) {
const allScenarios = loadScenarios();
const scenario = allScenarios.find(s => s.id === input.scenario.id);
if (!scenario) {
return {
scenarioId: input.scenario.id,
status: "fail",
score: 0,
summary: "Сценарий не найден",
rawLog: "error=Scenario not found"
};
}
const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true });
const inference = helpers.getInferenceEndpoint ? helpers.getInferenceEndpoint(input.model.id) : null;
const rawBaseUrl = (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) || "http://127.0.0.1:11434/v1";
const baseUrl = rawBaseUrl.replace(/\/+$/, "");
const apiKey = provider?.apiKey || "none";
const requestBody = {
model: input.model.model,
messages: [{ role: "user", content: scenario.prompt }],
temperature: 0
};
const response = await fetch(`${baseUrl}/chat/completions`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`
},
body: JSON.stringify(requestBody)
});
const data = await response.json();
const text = (data.choices?.[0]?.message?.content || "").trim();
const result = validateResponse(scenario, text);
const rawLog = [
`model=${input.model.model}`,
`scenario=${scenario.id} ${scenario.name}`,
`user_turn_1=${scenario.prompt}`,
`assistant_turn_1=${text}`,
`final_answer=${text}`,
`verdict=${result.pass ? "pass" : "fail"}`,
`score=${result.pass ? 100 : 0}`,
`summary=${result.reason}`,
`\n--- RAW API RESPONSE ---`,
JSON.stringify(data, null, 2)
].join("\n");
return {
scenarioId: scenario.id,
status: result.pass ? "pass" : "fail",
score: result.pass ? 100 : 0,
summary: result.reason,
rawLog: rawLog
};
},
async dispose() {}
};
}
export function scoreModelResults(results) {
const scored = results || [];
const total = scored.reduce((sum, r) => sum + (r.score || 0), 0);
const max = scored.length * 100;
return {
totalScore: max > 0 ? Math.round((total / max) * 100) : 0,
categories: []
};
}
export default {
manifest,
listScenarios,
getScenarios,
prepare,
createRuntime: prepare,
scoreModelResults
};