tianxuan / src /ml /gradient_monitor.ts
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
6.38 kB
/**
* 梯度监控器
* 记录和分析梯度范数,检测梯度爆炸/消失
*/
import * as fs from "fs";
import * as path from "path";
export class GradientMonitor {
private gradNormHistory: number[] = [];
private gradNormMax: number = 0;
private gradNormMin: number = Infinity;
private totalNorms: number = 0;
private stepCounts: number = 0;
private warningThreshold: number = 5.0;
private dangerousThreshold: number = 10.0;
// 日志文件
private logFile: string;
constructor(
private warningCallback: (msg: string) => void = (msg) => {
console.warn(`[梯度预警] ${msg}`);
},
private dangerCallback: (msg: string) => void = (msg) => {
console.error(`[梯度爆炸] ${msg}`);
},
logFile: string = "gradient_logs.json"
) {
this.logFile = logFile;
}
/**
* 记录梯度范数
*/
record(norm: number): void {
if (!Number.isFinite(norm)) {
this.warningCallback("收到非有限梯度范数: NaN/Inf");
return;
}
this.gradNormHistory.push(norm);
this.gradNormMax = Math.max(this.gradNormMax, norm);
this.gradNormMin = Math.min(this.gradNormMin, norm);
this.totalNorms += norm;
this.stepCounts++;
// 检查梯度异常
if (norm > this.dangerousThreshold) {
this.dangerCallback(`[梯度爆炸警告] 步数${this.stepCounts}: 范数=${norm.toFixed(4)} > 阈值${this.dangerousThreshold}`);
this.saveLog(); // 记录异常
} else if (norm > this.warningThreshold) {
this.warningCallback(`[梯度预警] 步数${this.stepCounts}: 范数=${norm.toFixed(4)} > 阈值${this.warningThreshold}`);
this.saveLog(); // 记录预警
} else if (norm < 1e-6 && norm > 0) {
this.warningCallback(`[梯度消失预警] 步数${this.stepCounts}: 范数=${norm.toExponential(4)}`);
}
}
/**
* 获取统计数据
*/
getStatistics(): {
avg: number;
max: number;
min: number;
median: number;
variance: number;
sampleCount: number;
gradNormHistory: number[];
} {
const avg = this.stepCounts > 0 ? this.totalNorms / this.stepCounts : 0;
const sorted = [...this.gradNormHistory].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
const variance = this.calculateVariance(avg);
return {
avg,
max: this.gradNormMax,
min: this.gradNormMin === Infinity ? 0 : this.gradNormMin,
median,
variance,
sampleCount: this.stepCounts,
gradNormHistory: [...this.gradNormHistory],
};
}
/**
* 报告当前状态
*/
report(): void {
const stats = this.getStatistics();
if (stats.sampleCount === 0) {
console.log("\n=== 梯度监控报告(空数据)===\n");
return;
}
console.log("\n========== 梯度监控报告 ==========");
console.log(`训练步数: ${stats.sampleCount}`);
console.log(`平均梯度范数: ${stats.avg.toFixed(6)}`);
console.log(`最大梯度范数: ${stats.max.toFixed(6)}`);
console.log(`最小梯度范数: ${stats.min.toFixed(6)}`);
console.log(`中位梯度范数: ${stats.median.toFixed(6)}`);
console.log(`梯度范数方差: ${stats.variance.toFixed(6)}`);
// 评估标准
let status = "正常";
if (stats.max > this.dangerousThreshold) status = "危险 (严重梯度爆炸!)";
else if (stats.max > this.warningThreshold) status = "预警 (梯度膨胀)";
else if (stats.avg < 1e-4) status = "异常 (梯度消失!)";
else if (stats.avg < 0.01) status = "稳定" + (stats.variance > 0.1 ? " (高变异性)" : "");
console.log(`\n评估结果:${status}`);
if (stats.max > 1.0) {
const quality = stats.max > 10.0 ? "严重" : "需要";
console.log(`[系统] 梯度${quality}裁剪:最大梯度=${stats.max.toFixed(4)}`);
}
console.log(`\n建议: 理想梯度范数范围 0.1-1.0,方差 < 0.2`);
console.log("======================================\n");
this.saveLog();
}
/**
* 计算方差
*/
private calculateVariance(mean: number): number {
if (this.gradNormHistory.length < 2) return 0;
const sumSquaredDiff = this.gradNormHistory.reduce((sum, norm) => {
return sum + Math.pow(norm - mean, 2);
}, 0);
return sumSquaredDiff / this.gradNormHistory.length;
}
/**
* 保存日志
*/
saveLog(): void {
try {
const stats = this.getStatistics();
const data = {
timestamp: new Date().toISOString(),
gradientNorms: stats.gradNormHistory.slice(-1000), // 保存最近1000条
statistics: {
avg: stats.avg,
max: stats.max,
min: stats.min,
median: stats.median,
variance: stats.variance,
sampleCount: stats.sampleCount,
},
};
const filePath = path.resolve(this.logFile);
fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
} catch (e) {
// 静默异常
}
}
/**
* 重置监控器
*/
reset(): void {
this.gradNormHistory = [];
this.gradNormMax = 0;
this.gradNormMin = Infinity;
this.totalNorms = 0;
this.stepCounts = 0;
this.saveLog();
}
/**
* 加载历史日志
*/
loadHistory(): void {
try {
if (fs.existsSync(this.logFile)) {
const data = JSON.parse(fs.readFileSync(this.logFile, "utf-8"));
if (data.gradientNorms && Array.isArray(data.gradientNorms)) {
this.gradNormHistory = [...data.gradientNorms];
this.recalculateStats();
console.log(`[梯度监控] 已加载历史日志 (${this.stepCounts} 个样本)`);
}
}
} catch (e) {
// 静默异常
}
}
/**
* 根据历史数据重新计算统计
*/
private recalculateStats(): void {
if (this.gradNormHistory.length === 0) return;
this.gradNormMax = Math.max(...this.gradNormHistory);
this.gradNormMin = Math.min(...this.gradNormHistory);
this.totalNorms = this.gradNormHistory.reduce((a, b) => a + b, 0);
this.stepCounts = this.gradNormHistory.length;
}
/**
* 获取实时梯度范数(最后一条)
*/
getLatestGradNorm(): number {
return this.gradNormHistory.length > 0
? this.gradNormHistory[this.gradNormHistory.length - 1]
: 0;
}
}