#!/usr/bin/env node /** * 天玄生产训练脚本 * * 使用完整的 48MB 真实语料进行生产级训练 * 启用所有在线持续学习机制 */ import * as fs from "fs"; import * as path from "path"; import { spawn } from "child_process"; const CKPT_DIR = "data/checkpoints"; const OUT_DIR = "sandbox/ck-prod-final"; console.log("=== 天玄生产训练脚本 ===\n"); // 检查数据 const rawDir = "data/raw"; if (!fs.existsSync(rawDir)) { console.error("❌ 未找到 data/raw/ 目录"); process.exit(1); } const files = fs.readdirSync(rawDir).filter(f => f.endsWith(".txt")); let totalBytes = 0; for (const f of files) { const stat = fs.statSync(path.join(rawDir, f)); totalBytes += stat.size; } const totalChars = Math.floor(totalBytes / 2); // 近似中文字符数 console.log(`✅ 找到 ${files.length} 个训练文件,共 ${totalChars.toLocaleString()} 字符`); if (totalChars < 100000) { console.error("❌ 数据量太少 (< 100K 字符),无法有效训练"); process.exit(1); } // 构建训练命令 const args = [ "--epochs=5", "--realcorpus=1", `--realTokens=${Math.min(totalChars, 1000000)}`, // 最多 1M tokens `--out=${OUT_DIR}`, "--lr=0.002", "--emb=64", "--hidden=256", "--ctx=8", "--bptt=32", "--maxvocab=100", "--ckpt=20", "--warmupsteps=50", ]; console.log(`\n🚀 开始训练...`); console.log(` 输出目录: ${OUT_DIR}`); console.log(` 预计时间: ~30 分钟 (CPU)`); console.log(` 命令: npm run train -- ${args.join(" ")}\n`); // 执行训练 const proc = spawn("npm", ["run", "train", "--", ...args], { stdio: "inherit", shell: true, }); proc.on("close", (code) => { if (code === 0) { console.log("\n✅ 训练完成!"); console.log(` 检查点: ${OUT_DIR}`); // 验证检查点 const metaPath = path.join(OUT_DIR, "meta.json"); if (fs.existsSync(metaPath)) { const meta = JSON.parse(fs.readFileSync(metaPath, "utf8")); console.log(` 训练步骤: ${meta.t}`); console.log(` 训练字符: ${meta.corpusTokens.toLocaleString()}`); console.log(` 词表大小: ${meta.vocab.length}`); } // 运行诊断 console.log("\n🔍 运行质量诊断..."); const diagProc = spawn("node", ["sandbox/dist/ml/diagnose.js", OUT_DIR], { stdio: "inherit", shell: true, }); diagProc.on("close", (diagCode) => { process.exit(diagCode ?? 0); }); } else { console.error("\n❌ 训练失败"); process.exit(code ?? 1); } });