#!/usr/bin/env node /** * 天玄 TianXuan - 集成训练监控器 * 监控训练进度和感受模块状态 */ const fs = require('fs'); const path = require('path'); const CHECKPOINT_DIR = process.argv[2] || 'data/checkpoints_v4'; const META_PATH = path.join(CHECKPOINT_DIR, 'meta.json'); const TARGET_STEPS = 2000; console.log('='.repeat(70)); console.log('天玄 TianXuan - 集成训练监控器'); console.log('='.repeat(70)); console.log(`监控目录:${CHECKPOINT_DIR}`); console.log(`目标步数:${TARGET_STEPS}`); console.log('按 Ctrl+C 停止监控\n'); let lastBatch = 0; let startTime = Date.now(); function getMeta() { if (!fs.existsSync(META_PATH)) return null; return JSON.parse(fs.readFileSync(META_PATH, 'utf8')); } function formatTime(ms) { const seconds = Math.floor(ms / 1000); const minutes = Math.floor(seconds / 60); const hours = Math.floor(minutes / 60); if (hours > 0) return `${hours}小时${minutes % 60}分钟${seconds % 60}秒`; if (minutes > 0) return `${minutes}分钟${seconds % 60}秒`; return `${seconds}秒`; } function poll() { const meta = getMeta(); if (meta) { const currentBatch = meta.t || 0; if (currentBatch !== lastBatch) { if (lastBatch > 0) { const elapsed = Date.now() - startTime; const bps = (currentBatch - lastBatch) / (elapsed / 1000); console.log(`\n>>> 进度更新:${lastBatch} → ${currentBatch} 步 (${bps.toFixed(1)} 步/秒)`); } lastBatch = currentBatch; const progress = Math.min(100, (currentBatch / TARGET_STEPS) * 100); const elapsed = Date.now() - startTime; const bps = currentBatch / (elapsed / 1000); const etaSeconds = (TARGET_STEPS - currentBatch) / bps; console.log(`\n【训练状态】`); console.log(` 进度:${currentBatch} / ${TARGET_STEPS} (${progress.toFixed(1)}%)`); console.log(` 语料长度:${(meta.corpusTokens || 0).toLocaleString()} 字符`); console.log(` 训练速度:${bps.toFixed(2)} 步/秒`); console.log(` 已运行:${formatTime(elapsed)}`); console.log(` 预计剩余:${formatTime(etaSeconds * 1000)}`); console.log(''); console.log('【在线学习机制】'); console.log(` STDP: ${meta.stdp ? '✓' : '✗'} ${meta.stdpState ? `(已更新 ${meta.stdpState.stepCount} 次)` : ''}`); console.log(` STDA: ${meta.stda ? '✓' : '✗'}`); console.log(` Titans: ${meta.onlineTitans ? '✓' : '✗'}`); console.log(` Replay: ${meta.replay ? '✓' : '✗'}`); // 检查是否完成 if (currentBatch >= TARGET_STEPS) { console.log('\n✅ 已达到目标步数!'); process.exit(0); } } } else { console.log(`\n等待检查点创建... (${new Date().toLocaleTimeString('zh-CN')})`); } } // 初始状态 poll(); // 启动轮询 const interval = setInterval(poll, 10000); // 处理中断 process.on('SIGINT', () => { console.log('\n\n监控已停止'); clearInterval(interval); process.exit(0); });