File size: 3,072 Bytes
db7bc06
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
#!/usr/bin/env node
/**
 * 天玄 TianXuan - 集成训练监控器
 * 监控训练进度和感受模块状态
 */

const fs = require('fs');
const path = require('path');

const CHECKPOINT_DIR = process.argv[2] || 'data/checkpoints_v4';
const META_PATH = path.join(CHECKPOINT_DIR, 'meta.json');
const TARGET_STEPS = 2000;

console.log('='.repeat(70));
console.log('天玄 TianXuan - 集成训练监控器');
console.log('='.repeat(70));
console.log(`监控目录:${CHECKPOINT_DIR}`);
console.log(`目标步数:${TARGET_STEPS}`);
console.log('按 Ctrl+C 停止监控\n');

let lastBatch = 0;
let startTime = Date.now();

function getMeta() {
  if (!fs.existsSync(META_PATH)) return null;
  return JSON.parse(fs.readFileSync(META_PATH, 'utf8'));
}

function formatTime(ms) {
  const seconds = Math.floor(ms / 1000);
  const minutes = Math.floor(seconds / 60);
  const hours = Math.floor(minutes / 60);
  if (hours > 0) return `${hours}小时${minutes % 60}分钟${seconds % 60}秒`;
  if (minutes > 0) return `${minutes}分钟${seconds % 60}秒`;
  return `${seconds}秒`;
}

function poll() {
  const meta = getMeta();
  
  if (meta) {
    const currentBatch = meta.t || 0;
    
    if (currentBatch !== lastBatch) {
      if (lastBatch > 0) {
        const elapsed = Date.now() - startTime;
        const bps = (currentBatch - lastBatch) / (elapsed / 1000);
        console.log(`\n>>> 进度更新:${lastBatch} → ${currentBatch} 步 (${bps.toFixed(1)} 步/秒)`);
      }
      lastBatch = currentBatch;
      
      const progress = Math.min(100, (currentBatch / TARGET_STEPS) * 100);
      const elapsed = Date.now() - startTime;
      const bps = currentBatch / (elapsed / 1000);
      const etaSeconds = (TARGET_STEPS - currentBatch) / bps;
      
      console.log(`\n【训练状态】`);
      console.log(`  进度:${currentBatch} / ${TARGET_STEPS} (${progress.toFixed(1)}%)`);
      console.log(`  语料长度:${(meta.corpusTokens || 0).toLocaleString()} 字符`);
      console.log(`  训练速度:${bps.toFixed(2)} 步/秒`);
      console.log(`  已运行:${formatTime(elapsed)}`);
      console.log(`  预计剩余:${formatTime(etaSeconds * 1000)}`);
      console.log('');
      console.log('【在线学习机制】');
      console.log(`  STDP: ${meta.stdp ? '✓' : '✗'} ${meta.stdpState ? `(已更新 ${meta.stdpState.stepCount} 次)` : ''}`);
      console.log(`  STDA: ${meta.stda ? '✓' : '✗'}`);
      console.log(`  Titans: ${meta.onlineTitans ? '✓' : '✗'}`);
      console.log(`  Replay: ${meta.replay ? '✓' : '✗'}`);
      
      // 检查是否完成
      if (currentBatch >= TARGET_STEPS) {
        console.log('\n✅ 已达到目标步数!');
        process.exit(0);
      }
    }
  } else {
    console.log(`\n等待检查点创建... (${new Date().toLocaleTimeString('zh-CN')})`);
  }
}

// 初始状态
poll();

// 启动轮询
const interval = setInterval(poll, 10000);

// 处理中断
process.on('SIGINT', () => {
  console.log('\n\n监控已停止');
  clearInterval(interval);
  process.exit(0);
});