/** * 准备训练数据:将清理后的语料放入 data/raw/ 供训练脚本使用 */ import * as fs from "fs"; import * as path from "path"; function main() { const rawDir = "data/raw"; // 确保 raw 目录存在 if (!fs.existsSync(rawDir)) { fs.mkdirSync(rawDir, { recursive: true }); } // 读取清理后的语料(从 large 版本) const largePath = "data/corpus_large.txt"; console.log(`读取: ${largePath}`); const raw = fs.readFileSync(largePath, "utf-8"); console.log(`原始字符数: ${raw.length}`); // 清理非CJK字符 const cleaned = raw.replace(/[^\u4E00-\u9FFF\u3000-\u303F\uFF00-\uFFEF\u2014\u2018-\u201D\u2026\n\r]/g, ""); console.log(`清理后字符数: ${cleaned.length}`); // 分成多个文件(每个约 30KB,符合训练脚本的 rawPerFile 限制) const chunkSize = 30_000; let fileIndex = 0; let offset = 0; while (offset < cleaned.length) { const chunk = cleaned.slice(offset, offset + chunkSize); const fileName = `event_${String(fileIndex).padStart(3, '0')}.txt`; const filePath = path.join(rawDir, fileName); fs.writeFileSync(filePath, chunk, "utf-8"); console.log(` 写入: ${filePath} (${chunk.length} 字符)`); fileIndex++; offset += chunkSize; } // 同时复制一个到 corpus.txt(供 --resume 使用) fs.writeFileSync("data/corpus.txt", cleaned, "utf-8"); console.log(`\n总文件数: ${fileIndex} 个 (在 ${rawDir}/)`); console.log(`已创建: data/corpus.txt (${cleaned.length} 字符)`); } main();