File size: 1,155 Bytes
db7bc06
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
const fs = require("fs");
const path = require("path");

const corpusPath = path.join(process.cwd(), "data", "corpus.txt");
const expectedLength = 137377;

if (!fs.existsSync(corpusPath)) {
    console.error("corpus.txt not found");
    process.exit(1);
}

const currentSize = fs.statSync(corpusPath).size;
console.log("Current corpus size:", currentSize, "chars");
console.log("Expected corpus size:", expectedLength, "chars");

if (currentSize === expectedLength) {
    console.log("Size matches, no truncation needed");
    process.exit(0);
}

if (currentSize < expectedLength) {
    console.log("Corpus is too small, need to regenerate");
    process.exit(2);
}

// Read text and truncate
const text = fs.readFileSync(corpusPath, "utf-8");
console.log("Read", text.length, "chars from disk");

if (text.length !== currentSize) {
    console.warn("Note: file size differs from char length (BOM or encoding), using char count");
}

const truncated = text.slice(0, expectedLength);
fs.writeFileSync(corpusPath, truncated, "utf-8");
const newSize = fs.statSync(corpusPath).size;
console.log("Truncated to:", truncated.length, "chars,", newSize, "bytes");