# Truncate corpus.txt to match checkpoint expected length (137377) $targetPath = "data/corpus.txt" $expectedLength = 137377 if (-not (Test-Path $targetPath)) { Write-Output "corpus.txt not found" exit 1 } $currentLength = (Get-Item $targetPath).Length Write-Output "Current corpus size: $currentLength chars" Write-Output "Expected corpus size: $expectedLength chars" if ($currentLength -eq $expectedLength) { Write-Output "Size matches, no truncation needed" exit 0 } if ($currentLength -lt $expectedLength) { Write-Output "Corpus is too small, need to regenerate" exit 2 } # Read as binary, truncate, write back $bytes = [System.IO.File]::ReadAllBytes($targetPath) $utf8 = New-Object System.Text.UTF8Encoding $false $text = $utf8.GetString($bytes) # Ensure we don't cut in the middle of a multi-byte char while ($text.Length -gt $expectedLength) { $text = $text.Substring(0, $text.Length - 1) } # Find last newline or just take exact length if ($text.Length -ne $expectedLength) { $text = $text.Substring(0, $expectedLength) } [System.IO.File]::WriteAllText($targetPath, $text, [System.Text.Encoding]::UTF8) $newLength = (Get-Item $targetPath).Length Write-Output "Truncated to: $newLength chars"