#!/usr/bin/env node /** * 天玄 TianXuan - 训练监控器 * 实时监控训练进度和在线学习指标 */ const fs = require('fs'); const path = require('path'); const { execSync } = require('child_process'); const CHECKPOINT_DIR = process.argv[2] || 'data/checkpoints_v3'; const POLL_INTERVAL = 10000; // 10秒轮询 console.log('='.repeat(70)); console.log('天玄 TianXuan - 训练监控器'); console.log('='.repeat(70)); console.log(`监控目录:${CHECKPOINT_DIR}`); console.log(`轮询间隔:${POLL_INTERVAL / 1000} 秒`); console.log('按 Ctrl+C 停止监控\n'); let lastBatch = 0; let lastLoss = null; let startTime = Date.now(); function getMeta() { const metaPath = path.join(CHECKPOINT_DIR, 'meta.json'); if (!fs.existsSync(metaPath)) return null; return JSON.parse(fs.readFileSync(metaPath, 'utf8')); } function getModelSize() { const modelPath = path.join(CHECKPOINT_DIR, 'model.json'); if (!fs.existsSync(modelPath)) return 0; return fs.statSync(modelPath).size; } function formatTime(ms) { const seconds = Math.floor(ms / 1000); const minutes = Math.floor(seconds / 60); const hours = Math.floor(minutes / 60); if (hours > 0) return `${hours}小时${minutes % 60}分钟`; if (minutes > 0) return `${minutes}分钟${seconds % 60}秒`; return `${seconds}秒`; } function calculateETA(meta) { if (!meta || lastBatch === 0) return 'N/A'; const elapsed = Date.now() - startTime; const batchesPerSecond = lastBatch / (elapsed / 1000); // 预估总批次 const corpusSize = meta.corpusTokens || 5000000; const seqLength = meta.config?.bptt || 32; const batchSize = 16; const totalBatches = Math.floor((corpusSize / seqLength) * (meta.epochs || 10) / batchSize); const remaining = totalBatches - lastBatch; const remainingSeconds = remaining / batchesPerSecond; return formatTime(remainingSeconds * 1000); } function displayStatus(meta) { if (!meta) return; const currentBatch = meta.t || 0; const elapsed = Date.now() - startTime; const bps = currentBatch / (elapsed / 1000); console.log('\n' + '─'.repeat(70)); console.log(`时间:${new Date().toLocaleString('zh-CN')}`); console.log(`训练步数:${currentBatch}`); console.log(`语料长度:${(meta.corpusTokens || 0).toLocaleString()} 字符`); console.log(`词表大小:${meta.config?.vocabSize || 0}`); console.log(`训练速度:${bps.toFixed(2)} 步/秒`); console.log(`已运行:${formatTime(elapsed)}`); console.log(`预计剩余:${calculateETA(meta)}`); console.log(''); console.log('【在线学习机制】'); console.log(` STDP: ${meta.stdp ? '✓ 已启用' : '✗ 未启用'}`); console.log(` STDA: ${meta.stda ? '✓ 已启用' : '✗ 未启用'}`); console.log(` Titans: ${meta.onlineTitans ? '✓ 已启用' : '✗ 未启用'}`); console.log(` Replay: ${meta.replay ? '✓ 已启用' : '✗ 未启用'}`); console.log(''); if (meta.stdpState) { console.log('【STDP 状态】'); console.log(` 更新步骤:${meta.stdpState.stepCount}`); console.log(` LTP 信号:${meta.stdpState.ltpSignalSum.toFixed(4)}`); console.log(` LTD 信号:${meta.stdpState.ltdSignalSum.toFixed(4)}`); console.log(''); } if (meta.stdaState) { console.log('【STDA 状态】'); console.log(` 阈值偏移:${meta.stdaState.thresholdShift.toFixed(4)}`); console.log(` 平均激活:${meta.stdaState.actAvg.toFixed(4)}`); console.log(''); } if (meta.replayState) { console.log('【Replay Buffer】'); console.log(` 当前大小:${meta.replayState.buffer?.length || 0} / ${meta.replayState.maxCapacity}`); console.log(` 占用率:${((meta.replayState.buffer?.length || 0) / meta.replayState.maxCapacity * 100).toFixed(1)}%`); console.log(''); } // 检查是否有新的检查点保存 const modelSize = getModelSize(); console.log(`【检查点】`); console.log(` 模型大小:${(modelSize / 1024 / 1024).toFixed(2)} MB`); console.log(` 最后更新:${new Date(fs.statSync(path.join(CHECKPOINT_DIR, 'meta.json')).mtime).toLocaleString('zh-CN')}`); } function poll() { const meta = getMeta(); if (meta) { const currentBatch = meta.t || 0; if (currentBatch !== lastBatch) { if (lastBatch > 0) { console.log(`\n>>> 进度更新:批次 ${lastBatch} → ${currentBatch}`); } lastBatch = currentBatch; displayStatus(meta); } } else { console.log(`\n等待检查点创建... (${new Date().toLocaleTimeString('zh-CN')})`); } } // 初始状态 poll(); // 启动轮询 const interval = setInterval(poll, POLL_INTERVAL); // 处理中断 process.on('SIGINT', () => { console.log('\n\n监控已停止'); clearInterval(interval); process.exit(0); });