/** * 语义反思层:在字符级模型输出基础上,增加"语言结构感" * 通过 n-gram 约束 + 标点控制 + 语义连贯性检查,修复字符级模型的"乱码"问题 */ import * as fs from "fs"; import * as path from "path"; /** 高频搭配矩阵:记录字符对 (bigram) 的共现频率 */ interface BigramMatrix { matrix: Map; // key: "char1|char2", value: count total: number; charFreq: Map; } /** 反思规则配置 */ interface ReflectionConfig { /** 最小句长(字符数),低于此值不强制结束 */ minSentenceLen: number; /** 最大句长,超过强制结束 */ maxSentenceLen: number; /** 最大连续重复字符数 */ maxRepeatCount: number; /** bigram 概率阈值,低于此值时压制 */ bigramThreshold: number; /** 标点概率提升权重 */ punctWeight: number; /** 是否启用反思层 */ enabled: boolean; } const DEFAULT_CONFIG: ReflectionConfig = { minSentenceLen: 5, maxSentenceLen: 25, maxRepeatCount: 2, bigramThreshold: 0.001, punctWeight: 5.0, enabled: true, }; /** * 构建 n-gram 统计 */ function buildBigramStats(text: string): BigramMatrix { const matrix = new Map(); const charFreq = new Map(); let total = 0; for (let i = 0; i < text.length - 1; i++) { const a = text[i]; const b = text[i + 1]; // 统计 char-bigram const key = `${a}|${b}`; matrix.set(key, (matrix.get(key) ?? 0) + 1); total++; // 统计单字符频率 charFreq.set(a, (charFreq.get(a) ?? 0) + 1); } // 最后一个字符 if (text.length > 0) { const last = text[text.length - 1]; charFreq.set(last, (charFreq.get(last) ?? 0) + 1); } return { matrix, total, charFreq }; } /** * 获取 bigram 条件概率 P(b|a) */ function bigramProb(matrix: BigramMatrix, a: string, b: string): number { const count = matrix.matrix.get(`${a}|${b}`) ?? 0; const freqA = matrix.charFreq.get(a) ?? 0; if (freqA === 0) return 0; return count / freqA; } /** * 反思层:检查并修正生成的字符 * @param char 待生成的字符 * @param context 已生成的上下文(最近 maxSentenceLen 字符) * @param config 配置 * @param bigramStats n-gram 统计 */ export function reflect( char: string, context: string, config: ReflectionConfig = DEFAULT_CONFIG, bigramStats?: BigramMatrix, ): { char: string; action: 'accept' | 'reject' | 'force_punct' | 'force_end'; reason: string; } { if (!config.enabled) { return { char, action: 'accept', reason: 'reflection disabled' }; } // 规则1: 连续重复检测 if (context.length >= config.maxRepeatCount) { const suffix = context.slice(-config.maxRepeatCount); if (suffix.split('').every(c => c === char)) { // 强制插入句号,打破循环 return { char: '。', action: 'force_end', reason: `repeat break: ${char.repeat(config.maxRepeatCount + 1)}` }; } } // 规则2: 句长控制 const currentSentenceLen = getCurrentSentenceLen(context); if (currentSentenceLen >= config.maxSentenceLen) { return { char: '。', action: 'force_end', reason: `sentence too long: ${currentSentenceLen}` }; } // 规则3: bigram 概率检测 if (bigramStats && context.length > 0) { const lastChar = context[context.length - 1]; const prob = bigramProb(bigramStats, lastChar, char); if (prob < config.bigramThreshold && char !== '。' && char !== ',' && char !== '!' && char !== '?') { // 低概率组合,但有概率接受(模拟语言的不确定性) if (Math.random() < 0.3) { return { char, action: 'accept', reason: `low prob bigram: ${lastChar}${char}(${prob.toFixed(4)})` }; } // 否则尝试替换为更高概率的字符 const forced = findBetterChar(context, bigramStats, config); if (forced) { return { char: forced, action: 'reject', reason: `low prob replaced: ${lastChar}${char}(${prob.toFixed(4)}) -> ${forced}` }; } } } // 规则4: 标点概率提升 if (char === '。' || char === ',' || char === '!' || char === '?') { return { char, action: 'accept', reason: 'punctuation' }; } return { char, action: 'accept', reason: 'normal' }; } /** * 获取当前句子的长度(从最后一个标点之后开始计算) */ function getCurrentSentenceLen(context: string): number { let len = 0; for (let i = context.length - 1; i >= 0; i--) { if (['。', '!', '?'].includes(context[i])) break; len++; } return len; } /** * 寻找更好的替代字符(基于 bigram 概率) */ function findBetterChar(context: string, stats: BigramMatrix, config: ReflectionConfig): string | null { const lastChar = context[context.length - 1]; if (!lastChar) return null; // 收集所有可能的下一个字符及其概率 const candidates: Array<{ char: string; prob: number }> = []; const matrixEntries = Array.from(stats.matrix.entries()); for (const [key, count] of matrixEntries) { const [a, b] = key.split('|'); if (a === lastChar) { const freqA = stats.charFreq.get(a) ?? 0; if (freqA > 0) { candidates.push({ char: b, prob: count / freqA }); } } } // 按概率排序 candidates.sort((a, b) => b.prob - a.prob); // 选择概率最高的几个候选 const topCandidates = candidates.slice(0, 5); if (topCandidates.length === 0) return null; // 过滤掉标点,优先选择汉字 const hanziCandidates = topCandidates.filter(c => /^[\u4e00-\u9fa5]$/.test(c.char)); if (hanziCandidates.length > 0) { return hanziCandidates[0].char; } return topCandidates[0].char; } /** * 从语料构建 bigram 统计并保存 */ export function buildAndSaveBigrams(corpusPath: string, outputPath: string): void { const text = fs.readFileSync(corpusPath, 'utf-8'); const stats = buildBigramStats(text); // 转换为可序列化的格式 const data = { matrix: Array.from(stats.matrix.entries()), total: stats.total, charFreq: Array.from(stats.charFreq.entries()), }; fs.writeFileSync(outputPath, JSON.stringify(data, null, 2)); console.log(`Bigram stats saved: ${stats.total} bigrams, ${stats.charFreq.size} unique chars`); } /** * 加载 bigram 统计 */ export function loadBigrams(path: string): BigramMatrix | null { try { const data = JSON.parse(fs.readFileSync(path, 'utf-8')); return { matrix: new Map(data.matrix), total: data.total, charFreq: new Map(data.charFreq), }; } catch (e) { console.warn(`Failed to load bigrams from ${path}: ${e}`); return null; } }