File size: 3,560 Bytes
db7bc06
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
import * as fs from "fs";
import * as path from "path";
import { WorldCorpus } from "./corpus";

export interface DataSource {
  name: string;
  text: string;
}

/** 去掉 Project Gutenberg 头尾声明,保留正文 */
export function stripGutenberg(text: string): string {
  const startMark = /START OF (THE|THIS) PROJECT GUTENBERG EBOOK/i;
  const endMark = /END OF (THE|THIS) PROJECT GUTENBERG EBOOK/i;
  const lines = text.split(/\r?\n/);
  let started = false;
  const body: string[] = [];
  for (const line of lines) {
    if (!started && startMark.test(line)) {
      started = true;
      continue;
    }
    if (started && endMark.test(line)) break;
    if (started) body.push(line);
  }
  return body.join("\n").trim();
}

/** 只保留 CJK 汉字与中文标点(丢弃 ASCII 与内嵌 Gutenberg 贡献者注记) */
export function cleanCJK(text: string): string {
  return text.replace(/[^\u4E00-\u9FFF\u3000-\u303F\uFF00-\uFFEF\u2014\u2018-\u201D\u2026]/g, "");
}

/** 读取 data/raw 下的真实语料(*.txt) */
export function loadRealFiles(dir = "data/raw", maxPerFile = 40_000): DataSource[] {
  if (!fs.existsSync(dir)) return [];
  let entries: string[] = [];
  try {
    entries = fs.readdirSync(dir).filter((f) => f.endsWith(".txt"));
  } catch {
    return [];
  }
  const out: DataSource[] = [];
  for (const file of entries) {
    let raw: string;
    try {
      raw = fs.readFileSync(path.join(dir, file), "utf-8");
    } catch {
      continue; // 单个文件读失败不影响其他文件
    }
    // 只在有 Project Gutenberg 标记时才去除头尾声明
    const hasGutenberg = /START OF (THE|THIS) PROJECT GUTENBERG EBOOK/i.test(raw);
    const cleaned = cleanCJK(hasGutenberg ? stripGutenberg(raw) : raw).replace(/\s+/g, "");
    if (cleaned.length === 0) continue;
    out.push({ name: file, text: cleaned.slice(0, maxPerFile) });
  }
  return out;
}

export interface CorpusBuildOptions {
  /** 总字符预算 */
  tokens?: number;
  /** 合成语料(前 seed; 真实语料优先填满, 剩余用合成) */
  seed?: number;
  rawDir?: string;
  /** 每本真实书最多引入的字符数 */
  rawPerFile?: number;
}

export interface CorpusBundle {
  text: string;
  sources: string[];
}

/** 混合语料: 真实公版文本优先, 合成事件文本兜底补满预算 */
export function buildCorpus(options: CorpusBuildOptions = {}): CorpusBundle {
  const tokens = options.tokens ?? 150_000;
  const sources: string[] = [];
  let text = "";
  // 真实语料: 读不到目录/全部被清洗空 → 直接 fallback, 不抛 ENOENT
  try {
    const real = loadRealFiles(options.rawDir ?? "data/raw", options.rawPerFile ?? 30_000);
    for (const r of real) {
      text += r.text;
      sources.push(r.name);
    }
  } catch (e) {
    console.warn(`buildCorpus: 读真实语料失败 (${(e as Error).message}), 用合成语料兜底`);
  }
  if (text.length >= tokens) {
    return { text: text.slice(0, tokens), sources };
  }
  const synth = new WorldCorpus({ seed: options.seed ?? 7, tokens: tokens - text.length }).generate(tokens - text.length);
  text += synth;
  sources.push("synthetic");
  return { text: cleanCJK(text).slice(0, tokens), sources };
}

/** 切块: 返回带来源号的有序片段 */
export function chunkText(text: string, size = 64, overlap = 32): string[] {
  const chunks: string[] = [];
  for (let i = 0; i + size <= text.length; i += size - overlap) {
    chunks.push(text.slice(i, i + size));
  }
  if (chunks.length === 0 && text.length > 0) chunks.push(text);
  return chunks;
}