tianxuan / src /ml /reflection.ts
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
6.72 kB
/**
* 语义反思层:在字符级模型输出基础上,增加"语言结构感"
* 通过 n-gram 约束 + 标点控制 + 语义连贯性检查,修复字符级模型的"乱码"问题
*/
import * as fs from "fs";
import * as path from "path";
/** 高频搭配矩阵:记录字符对 (bigram) 的共现频率 */
interface BigramMatrix {
matrix: Map<string, number>; // key: "char1|char2", value: count
total: number;
charFreq: Map<string, number>;
}
/** 反思规则配置 */
interface ReflectionConfig {
/** 最小句长(字符数),低于此值不强制结束 */
minSentenceLen: number;
/** 最大句长,超过强制结束 */
maxSentenceLen: number;
/** 最大连续重复字符数 */
maxRepeatCount: number;
/** bigram 概率阈值,低于此值时压制 */
bigramThreshold: number;
/** 标点概率提升权重 */
punctWeight: number;
/** 是否启用反思层 */
enabled: boolean;
}
const DEFAULT_CONFIG: ReflectionConfig = {
minSentenceLen: 5,
maxSentenceLen: 25,
maxRepeatCount: 2,
bigramThreshold: 0.001,
punctWeight: 5.0,
enabled: true,
};
/**
* 构建 n-gram 统计
*/
function buildBigramStats(text: string): BigramMatrix {
const matrix = new Map<string, number>();
const charFreq = new Map<string, number>();
let total = 0;
for (let i = 0; i < text.length - 1; i++) {
const a = text[i];
const b = text[i + 1];
// 统计 char-bigram
const key = `${a}|${b}`;
matrix.set(key, (matrix.get(key) ?? 0) + 1);
total++;
// 统计单字符频率
charFreq.set(a, (charFreq.get(a) ?? 0) + 1);
}
// 最后一个字符
if (text.length > 0) {
const last = text[text.length - 1];
charFreq.set(last, (charFreq.get(last) ?? 0) + 1);
}
return { matrix, total, charFreq };
}
/**
* 获取 bigram 条件概率 P(b|a)
*/
function bigramProb(matrix: BigramMatrix, a: string, b: string): number {
const count = matrix.matrix.get(`${a}|${b}`) ?? 0;
const freqA = matrix.charFreq.get(a) ?? 0;
if (freqA === 0) return 0;
return count / freqA;
}
/**
* 反思层:检查并修正生成的字符
* @param char 待生成的字符
* @param context 已生成的上下文(最近 maxSentenceLen 字符)
* @param config 配置
* @param bigramStats n-gram 统计
*/
export function reflect(
char: string,
context: string,
config: ReflectionConfig = DEFAULT_CONFIG,
bigramStats?: BigramMatrix,
): {
char: string;
action: 'accept' | 'reject' | 'force_punct' | 'force_end';
reason: string;
} {
if (!config.enabled) {
return { char, action: 'accept', reason: 'reflection disabled' };
}
// 规则1: 连续重复检测
if (context.length >= config.maxRepeatCount) {
const suffix = context.slice(-config.maxRepeatCount);
if (suffix.split('').every(c => c === char)) {
// 强制插入句号,打破循环
return { char: '。', action: 'force_end', reason: `repeat break: ${char.repeat(config.maxRepeatCount + 1)}` };
}
}
// 规则2: 句长控制
const currentSentenceLen = getCurrentSentenceLen(context);
if (currentSentenceLen >= config.maxSentenceLen) {
return { char: '。', action: 'force_end', reason: `sentence too long: ${currentSentenceLen}` };
}
// 规则3: bigram 概率检测
if (bigramStats && context.length > 0) {
const lastChar = context[context.length - 1];
const prob = bigramProb(bigramStats, lastChar, char);
if (prob < config.bigramThreshold && char !== '。' && char !== ',' && char !== '!' && char !== '?') {
// 低概率组合,但有概率接受(模拟语言的不确定性)
if (Math.random() < 0.3) {
return { char, action: 'accept', reason: `low prob bigram: ${lastChar}${char}(${prob.toFixed(4)})` };
}
// 否则尝试替换为更高概率的字符
const forced = findBetterChar(context, bigramStats, config);
if (forced) {
return { char: forced, action: 'reject', reason: `low prob replaced: ${lastChar}${char}(${prob.toFixed(4)}) -> ${forced}` };
}
}
}
// 规则4: 标点概率提升
if (char === '。' || char === ',' || char === '!' || char === '?') {
return { char, action: 'accept', reason: 'punctuation' };
}
return { char, action: 'accept', reason: 'normal' };
}
/**
* 获取当前句子的长度(从最后一个标点之后开始计算)
*/
function getCurrentSentenceLen(context: string): number {
let len = 0;
for (let i = context.length - 1; i >= 0; i--) {
if (['。', '!', '?'].includes(context[i])) break;
len++;
}
return len;
}
/**
* 寻找更好的替代字符(基于 bigram 概率)
*/
function findBetterChar(context: string, stats: BigramMatrix, config: ReflectionConfig): string | null {
const lastChar = context[context.length - 1];
if (!lastChar) return null;
// 收集所有可能的下一个字符及其概率
const candidates: Array<{ char: string; prob: number }> = [];
const matrixEntries = Array.from(stats.matrix.entries());
for (const [key, count] of matrixEntries) {
const [a, b] = key.split('|');
if (a === lastChar) {
const freqA = stats.charFreq.get(a) ?? 0;
if (freqA > 0) {
candidates.push({ char: b, prob: count / freqA });
}
}
}
// 按概率排序
candidates.sort((a, b) => b.prob - a.prob);
// 选择概率最高的几个候选
const topCandidates = candidates.slice(0, 5);
if (topCandidates.length === 0) return null;
// 过滤掉标点,优先选择汉字
const hanziCandidates = topCandidates.filter(c => /^[\u4e00-\u9fa5]$/.test(c.char));
if (hanziCandidates.length > 0) {
return hanziCandidates[0].char;
}
return topCandidates[0].char;
}
/**
* 从语料构建 bigram 统计并保存
*/
export function buildAndSaveBigrams(corpusPath: string, outputPath: string): void {
const text = fs.readFileSync(corpusPath, 'utf-8');
const stats = buildBigramStats(text);
// 转换为可序列化的格式
const data = {
matrix: Array.from(stats.matrix.entries()),
total: stats.total,
charFreq: Array.from(stats.charFreq.entries()),
};
fs.writeFileSync(outputPath, JSON.stringify(data, null, 2));
console.log(`Bigram stats saved: ${stats.total} bigrams, ${stats.charFreq.size} unique chars`);
}
/**
* 加载 bigram 统计
*/
export function loadBigrams(path: string): BigramMatrix | null {
try {
const data = JSON.parse(fs.readFileSync(path, 'utf-8'));
return {
matrix: new Map(data.matrix),
total: data.total,
charFreq: new Map(data.charFreq),
};
} catch (e) {
console.warn(`Failed to load bigrams from ${path}: ${e}`);
return null;
}
}