tianxuan / src /ml /data.ts
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
3.56 kB
import * as fs from "fs";
import * as path from "path";
import { WorldCorpus } from "./corpus";
export interface DataSource {
name: string;
text: string;
}
/** 去掉 Project Gutenberg 头尾声明,保留正文 */
export function stripGutenberg(text: string): string {
const startMark = /START OF (THE|THIS) PROJECT GUTENBERG EBOOK/i;
const endMark = /END OF (THE|THIS) PROJECT GUTENBERG EBOOK/i;
const lines = text.split(/\r?\n/);
let started = false;
const body: string[] = [];
for (const line of lines) {
if (!started && startMark.test(line)) {
started = true;
continue;
}
if (started && endMark.test(line)) break;
if (started) body.push(line);
}
return body.join("\n").trim();
}
/** 只保留 CJK 汉字与中文标点(丢弃 ASCII 与内嵌 Gutenberg 贡献者注记) */
export function cleanCJK(text: string): string {
return text.replace(/[^\u4E00-\u9FFF\u3000-\u303F\uFF00-\uFFEF\u2014\u2018-\u201D\u2026]/g, "");
}
/** 读取 data/raw 下的真实语料(*.txt) */
export function loadRealFiles(dir = "data/raw", maxPerFile = 40_000): DataSource[] {
if (!fs.existsSync(dir)) return [];
let entries: string[] = [];
try {
entries = fs.readdirSync(dir).filter((f) => f.endsWith(".txt"));
} catch {
return [];
}
const out: DataSource[] = [];
for (const file of entries) {
let raw: string;
try {
raw = fs.readFileSync(path.join(dir, file), "utf-8");
} catch {
continue; // 单个文件读失败不影响其他文件
}
// 只在有 Project Gutenberg 标记时才去除头尾声明
const hasGutenberg = /START OF (THE|THIS) PROJECT GUTENBERG EBOOK/i.test(raw);
const cleaned = cleanCJK(hasGutenberg ? stripGutenberg(raw) : raw).replace(/\s+/g, "");
if (cleaned.length === 0) continue;
out.push({ name: file, text: cleaned.slice(0, maxPerFile) });
}
return out;
}
export interface CorpusBuildOptions {
/** 总字符预算 */
tokens?: number;
/** 合成语料(前 seed; 真实语料优先填满, 剩余用合成) */
seed?: number;
rawDir?: string;
/** 每本真实书最多引入的字符数 */
rawPerFile?: number;
}
export interface CorpusBundle {
text: string;
sources: string[];
}
/** 混合语料: 真实公版文本优先, 合成事件文本兜底补满预算 */
export function buildCorpus(options: CorpusBuildOptions = {}): CorpusBundle {
const tokens = options.tokens ?? 150_000;
const sources: string[] = [];
let text = "";
// 真实语料: 读不到目录/全部被清洗空 → 直接 fallback, 不抛 ENOENT
try {
const real = loadRealFiles(options.rawDir ?? "data/raw", options.rawPerFile ?? 30_000);
for (const r of real) {
text += r.text;
sources.push(r.name);
}
} catch (e) {
console.warn(`buildCorpus: 读真实语料失败 (${(e as Error).message}), 用合成语料兜底`);
}
if (text.length >= tokens) {
return { text: text.slice(0, tokens), sources };
}
const synth = new WorldCorpus({ seed: options.seed ?? 7, tokens: tokens - text.length }).generate(tokens - text.length);
text += synth;
sources.push("synthetic");
return { text: cleanCJK(text).slice(0, tokens), sources };
}
/** 切块: 返回带来源号的有序片段 */
export function chunkText(text: string, size = 64, overlap = 32): string[] {
const chunks: string[] = [];
for (let i = 0; i + size <= text.length; i += size - overlap) {
chunks.push(text.slice(i, i + size));
}
if (chunks.length === 0 && text.length > 0) chunks.push(text);
return chunks;
}