/** * 梯度监控器 * 记录和分析梯度范数,检测梯度爆炸/消失 */ import * as fs from "fs"; import * as path from "path"; export class GradientMonitor { private gradNormHistory: number[] = []; private gradNormMax: number = 0; private gradNormMin: number = Infinity; private totalNorms: number = 0; private stepCounts: number = 0; private warningThreshold: number = 5.0; private dangerousThreshold: number = 10.0; // 日志文件 private logFile: string; constructor( private warningCallback: (msg: string) => void = (msg) => { console.warn(`[梯度预警] ${msg}`); }, private dangerCallback: (msg: string) => void = (msg) => { console.error(`[梯度爆炸] ${msg}`); }, logFile: string = "gradient_logs.json" ) { this.logFile = logFile; } /** * 记录梯度范数 */ record(norm: number): void { if (!Number.isFinite(norm)) { this.warningCallback("收到非有限梯度范数: NaN/Inf"); return; } this.gradNormHistory.push(norm); this.gradNormMax = Math.max(this.gradNormMax, norm); this.gradNormMin = Math.min(this.gradNormMin, norm); this.totalNorms += norm; this.stepCounts++; // 检查梯度异常 if (norm > this.dangerousThreshold) { this.dangerCallback(`[梯度爆炸警告] 步数${this.stepCounts}: 范数=${norm.toFixed(4)} > 阈值${this.dangerousThreshold}`); this.saveLog(); // 记录异常 } else if (norm > this.warningThreshold) { this.warningCallback(`[梯度预警] 步数${this.stepCounts}: 范数=${norm.toFixed(4)} > 阈值${this.warningThreshold}`); this.saveLog(); // 记录预警 } else if (norm < 1e-6 && norm > 0) { this.warningCallback(`[梯度消失预警] 步数${this.stepCounts}: 范数=${norm.toExponential(4)}`); } } /** * 获取统计数据 */ getStatistics(): { avg: number; max: number; min: number; median: number; variance: number; sampleCount: number; gradNormHistory: number[]; } { const avg = this.stepCounts > 0 ? this.totalNorms / this.stepCounts : 0; const sorted = [...this.gradNormHistory].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; const variance = this.calculateVariance(avg); return { avg, max: this.gradNormMax, min: this.gradNormMin === Infinity ? 0 : this.gradNormMin, median, variance, sampleCount: this.stepCounts, gradNormHistory: [...this.gradNormHistory], }; } /** * 报告当前状态 */ report(): void { const stats = this.getStatistics(); if (stats.sampleCount === 0) { console.log("\n=== 梯度监控报告(空数据)===\n"); return; } console.log("\n========== 梯度监控报告 =========="); console.log(`训练步数: ${stats.sampleCount}`); console.log(`平均梯度范数: ${stats.avg.toFixed(6)}`); console.log(`最大梯度范数: ${stats.max.toFixed(6)}`); console.log(`最小梯度范数: ${stats.min.toFixed(6)}`); console.log(`中位梯度范数: ${stats.median.toFixed(6)}`); console.log(`梯度范数方差: ${stats.variance.toFixed(6)}`); // 评估标准 let status = "正常"; if (stats.max > this.dangerousThreshold) status = "危险 (严重梯度爆炸!)"; else if (stats.max > this.warningThreshold) status = "预警 (梯度膨胀)"; else if (stats.avg < 1e-4) status = "异常 (梯度消失!)"; else if (stats.avg < 0.01) status = "稳定" + (stats.variance > 0.1 ? " (高变异性)" : ""); console.log(`\n评估结果:${status}`); if (stats.max > 1.0) { const quality = stats.max > 10.0 ? "严重" : "需要"; console.log(`[系统] 梯度${quality}裁剪:最大梯度=${stats.max.toFixed(4)}`); } console.log(`\n建议: 理想梯度范数范围 0.1-1.0,方差 < 0.2`); console.log("======================================\n"); this.saveLog(); } /** * 计算方差 */ private calculateVariance(mean: number): number { if (this.gradNormHistory.length < 2) return 0; const sumSquaredDiff = this.gradNormHistory.reduce((sum, norm) => { return sum + Math.pow(norm - mean, 2); }, 0); return sumSquaredDiff / this.gradNormHistory.length; } /** * 保存日志 */ saveLog(): void { try { const stats = this.getStatistics(); const data = { timestamp: new Date().toISOString(), gradientNorms: stats.gradNormHistory.slice(-1000), // 保存最近1000条 statistics: { avg: stats.avg, max: stats.max, min: stats.min, median: stats.median, variance: stats.variance, sampleCount: stats.sampleCount, }, }; const filePath = path.resolve(this.logFile); fs.writeFileSync(filePath, JSON.stringify(data, null, 2)); } catch (e) { // 静默异常 } } /** * 重置监控器 */ reset(): void { this.gradNormHistory = []; this.gradNormMax = 0; this.gradNormMin = Infinity; this.totalNorms = 0; this.stepCounts = 0; this.saveLog(); } /** * 加载历史日志 */ loadHistory(): void { try { if (fs.existsSync(this.logFile)) { const data = JSON.parse(fs.readFileSync(this.logFile, "utf-8")); if (data.gradientNorms && Array.isArray(data.gradientNorms)) { this.gradNormHistory = [...data.gradientNorms]; this.recalculateStats(); console.log(`[梯度监控] 已加载历史日志 (${this.stepCounts} 个样本)`); } } } catch (e) { // 静默异常 } } /** * 根据历史数据重新计算统计 */ private recalculateStats(): void { if (this.gradNormHistory.length === 0) return; this.gradNormMax = Math.max(...this.gradNormHistory); this.gradNormMin = Math.min(...this.gradNormHistory); this.totalNorms = this.gradNormHistory.reduce((a, b) => a + b, 0); this.stepCounts = this.gradNormHistory.length; } /** * 获取实时梯度范数(最后一条) */ getLatestGradNorm(): number { return this.gradNormHistory.length > 0 ? this.gradNormHistory[this.gradNormHistory.length - 1] : 0; } }