File size: 4,817 Bytes
db7bc06
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
#!/usr/bin/env node
/**
 * 天玄 TianXuan - 训练监控器
 * 实时监控训练进度和在线学习指标
 */

const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

const CHECKPOINT_DIR = process.argv[2] || 'data/checkpoints_v3';
const POLL_INTERVAL = 10000; // 10秒轮询

console.log('='.repeat(70));
console.log('天玄 TianXuan - 训练监控器');
console.log('='.repeat(70));
console.log(`监控目录:${CHECKPOINT_DIR}`);
console.log(`轮询间隔:${POLL_INTERVAL / 1000} 秒`);
console.log('按 Ctrl+C 停止监控\n');

let lastBatch = 0;
let lastLoss = null;
let startTime = Date.now();

function getMeta() {
  const metaPath = path.join(CHECKPOINT_DIR, 'meta.json');
  if (!fs.existsSync(metaPath)) return null;
  return JSON.parse(fs.readFileSync(metaPath, 'utf8'));
}

function getModelSize() {
  const modelPath = path.join(CHECKPOINT_DIR, 'model.json');
  if (!fs.existsSync(modelPath)) return 0;
  return fs.statSync(modelPath).size;
}

function formatTime(ms) {
  const seconds = Math.floor(ms / 1000);
  const minutes = Math.floor(seconds / 60);
  const hours = Math.floor(minutes / 60);
  if (hours > 0) return `${hours}小时${minutes % 60}分钟`;
  if (minutes > 0) return `${minutes}分钟${seconds % 60}秒`;
  return `${seconds}秒`;
}

function calculateETA(meta) {
  if (!meta || lastBatch === 0) return 'N/A';
  
  const elapsed = Date.now() - startTime;
  const batchesPerSecond = lastBatch / (elapsed / 1000);
  
  // 预估总批次
  const corpusSize = meta.corpusTokens || 5000000;
  const seqLength = meta.config?.bptt || 32;
  const batchSize = 16;
  const totalBatches = Math.floor((corpusSize / seqLength) * (meta.epochs || 10) / batchSize);
  
  const remaining = totalBatches - lastBatch;
  const remainingSeconds = remaining / batchesPerSecond;
  
  return formatTime(remainingSeconds * 1000);
}

function displayStatus(meta) {
  if (!meta) return;
  
  const currentBatch = meta.t || 0;
  const elapsed = Date.now() - startTime;
  const bps = currentBatch / (elapsed / 1000);
  
  console.log('\n' + '─'.repeat(70));
  console.log(`时间:${new Date().toLocaleString('zh-CN')}`);
  console.log(`训练步数:${currentBatch}`);
  console.log(`语料长度:${(meta.corpusTokens || 0).toLocaleString()} 字符`);
  console.log(`词表大小:${meta.config?.vocabSize || 0}`);
  console.log(`训练速度:${bps.toFixed(2)} 步/秒`);
  console.log(`已运行:${formatTime(elapsed)}`);
  console.log(`预计剩余:${calculateETA(meta)}`);
  console.log('');
  console.log('【在线学习机制】');
  console.log(`  STDP: ${meta.stdp ? '✓ 已启用' : '✗ 未启用'}`);
  console.log(`  STDA: ${meta.stda ? '✓ 已启用' : '✗ 未启用'}`);
  console.log(`  Titans: ${meta.onlineTitans ? '✓ 已启用' : '✗ 未启用'}`);
  console.log(`  Replay: ${meta.replay ? '✓ 已启用' : '✗ 未启用'}`);
  console.log('');
  
  if (meta.stdpState) {
    console.log('【STDP 状态】');
    console.log(`  更新步骤:${meta.stdpState.stepCount}`);
    console.log(`  LTP 信号:${meta.stdpState.ltpSignalSum.toFixed(4)}`);
    console.log(`  LTD 信号:${meta.stdpState.ltdSignalSum.toFixed(4)}`);
    console.log('');
  }
  
  if (meta.stdaState) {
    console.log('【STDA 状态】');
    console.log(`  阈值偏移:${meta.stdaState.thresholdShift.toFixed(4)}`);
    console.log(`  平均激活:${meta.stdaState.actAvg.toFixed(4)}`);
    console.log('');
  }
  
  if (meta.replayState) {
    console.log('【Replay Buffer】');
    console.log(`  当前大小:${meta.replayState.buffer?.length || 0} / ${meta.replayState.maxCapacity}`);
    console.log(`  占用率:${((meta.replayState.buffer?.length || 0) / meta.replayState.maxCapacity * 100).toFixed(1)}%`);
    console.log('');
  }
  
  // 检查是否有新的检查点保存
  const modelSize = getModelSize();
  console.log(`【检查点】`);
  console.log(`  模型大小:${(modelSize / 1024 / 1024).toFixed(2)} MB`);
  console.log(`  最后更新:${new Date(fs.statSync(path.join(CHECKPOINT_DIR, 'meta.json')).mtime).toLocaleString('zh-CN')}`);
}

function poll() {
  const meta = getMeta();
  
  if (meta) {
    const currentBatch = meta.t || 0;
    
    if (currentBatch !== lastBatch) {
      if (lastBatch > 0) {
        console.log(`\n>>> 进度更新:批次 ${lastBatch} → ${currentBatch}`);
      }
      lastBatch = currentBatch;
      displayStatus(meta);
    }
  } else {
    console.log(`\n等待检查点创建... (${new Date().toLocaleTimeString('zh-CN')})`);
  }
}

// 初始状态
poll();

// 启动轮询
const interval = setInterval(poll, POLL_INTERVAL);

// 处理中断
process.on('SIGINT', () => {
  console.log('\n\n监控已停止');
  clearInterval(interval);
  process.exit(0);
});