File size: 1,557 Bytes
db7bc06
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
/**
 * 准备训练数据:将清理后的语料放入 data/raw/ 供训练脚本使用
 */
import * as fs from "fs";
import * as path from "path";

function main() {
  const rawDir = "data/raw";
  
  // 确保 raw 目录存在
  if (!fs.existsSync(rawDir)) {
    fs.mkdirSync(rawDir, { recursive: true });
  }
  
  // 读取清理后的语料(从 large 版本)
  const largePath = "data/corpus_large.txt";
  console.log(`读取: ${largePath}`);
  const raw = fs.readFileSync(largePath, "utf-8");
  console.log(`原始字符数: ${raw.length}`);
  
  // 清理非CJK字符
  const cleaned = raw.replace(/[^\u4E00-\u9FFF\u3000-\u303F\uFF00-\uFFEF\u2014\u2018-\u201D\u2026\n\r]/g, "");
  console.log(`清理后字符数: ${cleaned.length}`);
  
  // 分成多个文件(每个约 30KB,符合训练脚本的 rawPerFile 限制)
  const chunkSize = 30_000;
  let fileIndex = 0;
  let offset = 0;
  
  while (offset < cleaned.length) {
    const chunk = cleaned.slice(offset, offset + chunkSize);
    const fileName = `event_${String(fileIndex).padStart(3, '0')}.txt`;
    const filePath = path.join(rawDir, fileName);
    fs.writeFileSync(filePath, chunk, "utf-8");
    console.log(`  写入: ${filePath} (${chunk.length} 字符)`);
    fileIndex++;
    offset += chunkSize;
  }
  
  // 同时复制一个到 corpus.txt(供 --resume 使用)
  fs.writeFileSync("data/corpus.txt", cleaned, "utf-8");
  console.log(`\n总文件数: ${fileIndex} 个 (在 ${rawDir}/)`);
  console.log(`已创建: data/corpus.txt (${cleaned.length} 字符)`);
}

main();