tianxuan / scripts /fix-corpus.ps1
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
1.24 kB
# Truncate corpus.txt to match checkpoint expected length (137377)
$targetPath = "data/corpus.txt"
$expectedLength = 137377
if (-not (Test-Path $targetPath)) {
Write-Output "corpus.txt not found"
exit 1
}
$currentLength = (Get-Item $targetPath).Length
Write-Output "Current corpus size: $currentLength chars"
Write-Output "Expected corpus size: $expectedLength chars"
if ($currentLength -eq $expectedLength) {
Write-Output "Size matches, no truncation needed"
exit 0
}
if ($currentLength -lt $expectedLength) {
Write-Output "Corpus is too small, need to regenerate"
exit 2
}
# Read as binary, truncate, write back
$bytes = [System.IO.File]::ReadAllBytes($targetPath)
$utf8 = New-Object System.Text.UTF8Encoding $false
$text = $utf8.GetString($bytes)
# Ensure we don't cut in the middle of a multi-byte char
while ($text.Length -gt $expectedLength) {
$text = $text.Substring(0, $text.Length - 1)
}
# Find last newline or just take exact length
if ($text.Length -ne $expectedLength) {
$text = $text.Substring(0, $expectedLength)
}
[System.IO.File]::WriteAllText($targetPath, $text, [System.Text.Encoding]::UTF8)
$newLength = (Get-Item $targetPath).Length
Write-Output "Truncated to: $newLength chars"