tianxuan / scripts /prepare_training_data.ts
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
1.56 kB
/**
* 准备训练数据:将清理后的语料放入 data/raw/ 供训练脚本使用
*/
import * as fs from "fs";
import * as path from "path";
function main() {
const rawDir = "data/raw";
// 确保 raw 目录存在
if (!fs.existsSync(rawDir)) {
fs.mkdirSync(rawDir, { recursive: true });
}
// 读取清理后的语料(从 large 版本)
const largePath = "data/corpus_large.txt";
console.log(`读取: ${largePath}`);
const raw = fs.readFileSync(largePath, "utf-8");
console.log(`原始字符数: ${raw.length}`);
// 清理非CJK字符
const cleaned = raw.replace(/[^\u4E00-\u9FFF\u3000-\u303F\uFF00-\uFFEF\u2014\u2018-\u201D\u2026\n\r]/g, "");
console.log(`清理后字符数: ${cleaned.length}`);
// 分成多个文件(每个约 30KB,符合训练脚本的 rawPerFile 限制)
const chunkSize = 30_000;
let fileIndex = 0;
let offset = 0;
while (offset < cleaned.length) {
const chunk = cleaned.slice(offset, offset + chunkSize);
const fileName = `event_${String(fileIndex).padStart(3, '0')}.txt`;
const filePath = path.join(rawDir, fileName);
fs.writeFileSync(filePath, chunk, "utf-8");
console.log(` 写入: ${filePath} (${chunk.length} 字符)`);
fileIndex++;
offset += chunkSize;
}
// 同时复制一个到 corpus.txt(供 --resume 使用)
fs.writeFileSync("data/corpus.txt", cleaned, "utf-8");
console.log(`\n总文件数: ${fileIndex} 个 (在 ${rawDir}/)`);
console.log(`已创建: data/corpus.txt (${cleaned.length} 字符)`);
}
main();