tianxuan / scripts /fix-corpus.js
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
1.16 kB
const fs = require("fs");
const path = require("path");
const corpusPath = path.join(process.cwd(), "data", "corpus.txt");
const expectedLength = 137377;
if (!fs.existsSync(corpusPath)) {
console.error("corpus.txt not found");
process.exit(1);
}
const currentSize = fs.statSync(corpusPath).size;
console.log("Current corpus size:", currentSize, "chars");
console.log("Expected corpus size:", expectedLength, "chars");
if (currentSize === expectedLength) {
console.log("Size matches, no truncation needed");
process.exit(0);
}
if (currentSize < expectedLength) {
console.log("Corpus is too small, need to regenerate");
process.exit(2);
}
// Read text and truncate
const text = fs.readFileSync(corpusPath, "utf-8");
console.log("Read", text.length, "chars from disk");
if (text.length !== currentSize) {
console.warn("Note: file size differs from char length (BOM or encoding), using char count");
}
const truncated = text.slice(0, expectedLength);
fs.writeFileSync(corpusPath, truncated, "utf-8");
const newSize = fs.statSync(corpusPath).size;
console.log("Truncated to:", truncated.length, "chars,", newSize, "bytes");