tianxuan / scripts /monitor-training.js
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
4.82 kB
#!/usr/bin/env node
/**
* 天玄 TianXuan - 训练监控器
* 实时监控训练进度和在线学习指标
*/
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const CHECKPOINT_DIR = process.argv[2] || 'data/checkpoints_v3';
const POLL_INTERVAL = 10000; // 10秒轮询
console.log('='.repeat(70));
console.log('天玄 TianXuan - 训练监控器');
console.log('='.repeat(70));
console.log(`监控目录:${CHECKPOINT_DIR}`);
console.log(`轮询间隔:${POLL_INTERVAL / 1000} 秒`);
console.log('按 Ctrl+C 停止监控\n');
let lastBatch = 0;
let lastLoss = null;
let startTime = Date.now();
function getMeta() {
const metaPath = path.join(CHECKPOINT_DIR, 'meta.json');
if (!fs.existsSync(metaPath)) return null;
return JSON.parse(fs.readFileSync(metaPath, 'utf8'));
}
function getModelSize() {
const modelPath = path.join(CHECKPOINT_DIR, 'model.json');
if (!fs.existsSync(modelPath)) return 0;
return fs.statSync(modelPath).size;
}
function formatTime(ms) {
const seconds = Math.floor(ms / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
if (hours > 0) return `${hours}小时${minutes % 60}分钟`;
if (minutes > 0) return `${minutes}分钟${seconds % 60}秒`;
return `${seconds}秒`;
}
function calculateETA(meta) {
if (!meta || lastBatch === 0) return 'N/A';
const elapsed = Date.now() - startTime;
const batchesPerSecond = lastBatch / (elapsed / 1000);
// 预估总批次
const corpusSize = meta.corpusTokens || 5000000;
const seqLength = meta.config?.bptt || 32;
const batchSize = 16;
const totalBatches = Math.floor((corpusSize / seqLength) * (meta.epochs || 10) / batchSize);
const remaining = totalBatches - lastBatch;
const remainingSeconds = remaining / batchesPerSecond;
return formatTime(remainingSeconds * 1000);
}
function displayStatus(meta) {
if (!meta) return;
const currentBatch = meta.t || 0;
const elapsed = Date.now() - startTime;
const bps = currentBatch / (elapsed / 1000);
console.log('\n' + '─'.repeat(70));
console.log(`时间:${new Date().toLocaleString('zh-CN')}`);
console.log(`训练步数:${currentBatch}`);
console.log(`语料长度:${(meta.corpusTokens || 0).toLocaleString()} 字符`);
console.log(`词表大小:${meta.config?.vocabSize || 0}`);
console.log(`训练速度:${bps.toFixed(2)} 步/秒`);
console.log(`已运行:${formatTime(elapsed)}`);
console.log(`预计剩余:${calculateETA(meta)}`);
console.log('');
console.log('【在线学习机制】');
console.log(` STDP: ${meta.stdp ? '✓ 已启用' : '✗ 未启用'}`);
console.log(` STDA: ${meta.stda ? '✓ 已启用' : '✗ 未启用'}`);
console.log(` Titans: ${meta.onlineTitans ? '✓ 已启用' : '✗ 未启用'}`);
console.log(` Replay: ${meta.replay ? '✓ 已启用' : '✗ 未启用'}`);
console.log('');
if (meta.stdpState) {
console.log('【STDP 状态】');
console.log(` 更新步骤:${meta.stdpState.stepCount}`);
console.log(` LTP 信号:${meta.stdpState.ltpSignalSum.toFixed(4)}`);
console.log(` LTD 信号:${meta.stdpState.ltdSignalSum.toFixed(4)}`);
console.log('');
}
if (meta.stdaState) {
console.log('【STDA 状态】');
console.log(` 阈值偏移:${meta.stdaState.thresholdShift.toFixed(4)}`);
console.log(` 平均激活:${meta.stdaState.actAvg.toFixed(4)}`);
console.log('');
}
if (meta.replayState) {
console.log('【Replay Buffer】');
console.log(` 当前大小:${meta.replayState.buffer?.length || 0} / ${meta.replayState.maxCapacity}`);
console.log(` 占用率:${((meta.replayState.buffer?.length || 0) / meta.replayState.maxCapacity * 100).toFixed(1)}%`);
console.log('');
}
// 检查是否有新的检查点保存
const modelSize = getModelSize();
console.log(`【检查点】`);
console.log(` 模型大小:${(modelSize / 1024 / 1024).toFixed(2)} MB`);
console.log(` 最后更新:${new Date(fs.statSync(path.join(CHECKPOINT_DIR, 'meta.json')).mtime).toLocaleString('zh-CN')}`);
}
function poll() {
const meta = getMeta();
if (meta) {
const currentBatch = meta.t || 0;
if (currentBatch !== lastBatch) {
if (lastBatch > 0) {
console.log(`\n>>> 进度更新:批次 ${lastBatch} → ${currentBatch}`);
}
lastBatch = currentBatch;
displayStatus(meta);
}
} else {
console.log(`\n等待检查点创建... (${new Date().toLocaleTimeString('zh-CN')})`);
}
}
// 初始状态
poll();
// 启动轮询
const interval = setInterval(poll, POLL_INTERVAL);
// 处理中断
process.on('SIGINT', () => {
console.log('\n\n监控已停止');
clearInterval(interval);
process.exit(0);
});