tianxuan / scripts /prod-train.js
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
2.53 kB
#!/usr/bin/env node
/**
* 天玄生产训练脚本
*
* 使用完整的 48MB 真实语料进行生产级训练
* 启用所有在线持续学习机制
*/
import * as fs from "fs";
import * as path from "path";
import { spawn } from "child_process";
const CKPT_DIR = "data/checkpoints";
const OUT_DIR = "sandbox/ck-prod-final";
console.log("=== 天玄生产训练脚本 ===\n");
// 检查数据
const rawDir = "data/raw";
if (!fs.existsSync(rawDir)) {
console.error("❌ 未找到 data/raw/ 目录");
process.exit(1);
}
const files = fs.readdirSync(rawDir).filter(f => f.endsWith(".txt"));
let totalBytes = 0;
for (const f of files) {
const stat = fs.statSync(path.join(rawDir, f));
totalBytes += stat.size;
}
const totalChars = Math.floor(totalBytes / 2); // 近似中文字符数
console.log(`✅ 找到 ${files.length} 个训练文件,共 ${totalChars.toLocaleString()} 字符`);
if (totalChars < 100000) {
console.error("❌ 数据量太少 (< 100K 字符),无法有效训练");
process.exit(1);
}
// 构建训练命令
const args = [
"--epochs=5",
"--realcorpus=1",
`--realTokens=${Math.min(totalChars, 1000000)}`, // 最多 1M tokens
`--out=${OUT_DIR}`,
"--lr=0.002",
"--emb=64",
"--hidden=256",
"--ctx=8",
"--bptt=32",
"--maxvocab=100",
"--ckpt=20",
"--warmupsteps=50",
];
console.log(`\n🚀 开始训练...`);
console.log(` 输出目录: ${OUT_DIR}`);
console.log(` 预计时间: ~30 分钟 (CPU)`);
console.log(` 命令: npm run train -- ${args.join(" ")}\n`);
// 执行训练
const proc = spawn("npm", ["run", "train", "--", ...args], {
stdio: "inherit",
shell: true,
});
proc.on("close", (code) => {
if (code === 0) {
console.log("\n✅ 训练完成!");
console.log(` 检查点: ${OUT_DIR}`);
// 验证检查点
const metaPath = path.join(OUT_DIR, "meta.json");
if (fs.existsSync(metaPath)) {
const meta = JSON.parse(fs.readFileSync(metaPath, "utf8"));
console.log(` 训练步骤: ${meta.t}`);
console.log(` 训练字符: ${meta.corpusTokens.toLocaleString()}`);
console.log(` 词表大小: ${meta.vocab.length}`);
}
// 运行诊断
console.log("\n🔍 运行质量诊断...");
const diagProc = spawn("node", ["sandbox/dist/ml/diagnose.js", OUT_DIR], {
stdio: "inherit",
shell: true,
});
diagProc.on("close", (diagCode) => {
process.exit(diagCode ?? 0);
});
} else {
console.error("\n❌ 训练失败");
process.exit(code ?? 1);
}
});