const fs = require("fs"); const path = require("path"); const corpusPath = path.join(process.cwd(), "data", "corpus.txt"); const expectedLength = 137377; if (!fs.existsSync(corpusPath)) { console.error("corpus.txt not found"); process.exit(1); } const currentSize = fs.statSync(corpusPath).size; console.log("Current corpus size:", currentSize, "chars"); console.log("Expected corpus size:", expectedLength, "chars"); if (currentSize === expectedLength) { console.log("Size matches, no truncation needed"); process.exit(0); } if (currentSize < expectedLength) { console.log("Corpus is too small, need to regenerate"); process.exit(2); } // Read text and truncate const text = fs.readFileSync(corpusPath, "utf-8"); console.log("Read", text.length, "chars from disk"); if (text.length !== currentSize) { console.warn("Note: file size differs from char length (BOM or encoding), using char count"); } const truncated = text.slice(0, expectedLength); fs.writeFileSync(corpusPath, truncated, "utf-8"); const newSize = fs.statSync(corpusPath).size; console.log("Truncated to:", truncated.length, "chars,", newSize, "bytes");