| param( |
| [Parameter(Mandatory = $true)] |
| [string]$Manifest, |
| [Parameter(Mandatory = $true)] |
| [string]$Model, |
| [Parameter(Mandatory = $true)] |
| [string]$OutputRoot, |
| [string[]]$Seeds = @('0', '1', '2'), |
| [int]$Steps = 12000, |
| [int]$Images = 2500, |
| [int]$Layers = 3, |
| [int]$Size = 224, |
| [string]$EvaluationSplit = "validation" |
| ) |
|
|
| $ErrorActionPreference = "Stop" |
| $repo = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path |
| $manifestPath = (Resolve-Path $Manifest).Path |
| $modelPath = (Resolve-Path $Model).Path |
| $manifestHash = (Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant() |
| $modelHash = (Get-FileHash -LiteralPath $modelPath -Algorithm SHA256).Hash.ToLowerInvariant() |
| $outputPath = [System.IO.Path]::GetFullPath($OutputRoot) |
| New-Item -ItemType Directory -Force -Path $outputPath | Out-Null |
| $cargo = (Get-Command cargo).Source |
| $seedValues = @( |
| $Seeds | ForEach-Object { $_ -split ',' } | ForEach-Object { |
| $value = 0L |
| if (-not [long]::TryParse($_.Trim(), [ref]$value) -or $value -lt 0) { |
| throw "invalid seed: $_" |
| } |
| $value |
| } |
| ) |
| if ($seedValues.Count -eq 0 -or @($seedValues | Select-Object -Unique).Count -ne $seedValues.Count) { |
| throw 'provide at least one seed, without duplicates' |
| } |
|
|
| function Run-Cargo( |
| [string[]]$Arguments, |
| [string]$Stdout, |
| [string]$Stderr, |
| [string]$LogLevel |
| ) { |
| $env:RUST_LOG = $LogLevel |
| Push-Location $repo |
| $oldErrorPreference = $ErrorActionPreference |
| try { |
| |
| |
| |
| $ErrorActionPreference = "Continue" |
| & $cargo @Arguments 1> $Stdout 2> $Stderr |
| $exitCode = $LASTEXITCODE |
| } finally { |
| $ErrorActionPreference = $oldErrorPreference |
| Pop-Location |
| } |
| if ($exitCode -ne 0) { |
| throw "cargo exited with code $exitCode; see $Stdout and $Stderr" |
| } |
| } |
|
|
| foreach ($seed in $seedValues) { |
| $runDir = Join-Path $outputPath "${Layers}l${Size}-seed${seed}" |
| New-Item -ItemType Directory -Force -Path $runDir | Out-Null |
| $decoder = Join-Path $runDir "decoder.bin" |
| $training = Join-Path $runDir "training.json" |
| $quality = Join-Path $runDir "quality-imagenette-$EvaluationSplit.json" |
|
|
| if (-not ((Test-Path -LiteralPath $training) -and (Test-Path -LiteralPath $decoder))) { |
| Write-Output "training seed $seed at $(Get-Date -Format o)" |
| Run-Cargo ` |
| -Arguments @( |
| "run", "--release", "--example", "train_decoder", "--", |
| $manifestPath, $modelPath, "$Steps", "$Images", "$Layers", "$Size", "$seed", $runDir |
| ) ` |
| -Stdout (Join-Path $runDir "training.stdout.log") ` |
| -Stderr (Join-Path $runDir "training.stderr.log") ` |
| -LogLevel "info" |
| } else { |
| Write-Output "seed $seed training outputs already exist; preserving them" |
| } |
|
|
| $record = Get-Content -Raw -LiteralPath $training | ConvertFrom-Json |
| if ( |
| $record.seed -ne $seed -or |
| $record.steps -ne $Steps -or |
| $record.requested_images -ne $Images -or |
| $record.encoder_layers -ne $Layers -or |
| $record.image_size -ne $Size -or |
| $record.dataset_manifest_sha256 -ne $manifestHash -or |
| $record.model_sha256 -ne $modelHash |
| ) { |
| throw "existing training record does not match requested cell: $training" |
| } |
|
|
| if (-not (Test-Path -LiteralPath $quality)) { |
| Write-Output "evaluating seed $seed at $(Get-Date -Format o)" |
| Run-Cargo ` |
| -Arguments @( |
| "run", "--release", "--example", "evaluate_decoder", "--", |
| "--manifest", $manifestPath, |
| "--model", $modelPath, |
| "--decoder", $decoder, |
| "--split", $EvaluationSplit, |
| "--layers", "$Layers", |
| "--size", "$Size", |
| "--output", $quality, |
| "--samples", "12" |
| ) ` |
| -Stdout (Join-Path $runDir "evaluation.stdout.log") ` |
| -Stderr (Join-Path $runDir "evaluation.stderr.log") ` |
| -LogLevel "warn" |
| } else { |
| Write-Output "seed $seed quality output already exists; preserving it" |
| } |
|
|
| $qualityRecord = Get-Content -Raw -LiteralPath $quality | ConvertFrom-Json |
| if ( |
| $qualityRecord.split -ne $EvaluationSplit -or |
| $qualityRecord.encoder_layers -ne $Layers -or |
| $qualityRecord.image_size -ne $Size -or |
| $qualityRecord.manifest_sha256 -ne $manifestHash -or |
| $qualityRecord.model_sha256 -ne $modelHash -or |
| $qualityRecord.decoder_sha256 -ne $record.decoder_sha256 |
| ) { |
| throw "quality record does not match its training cell: $quality" |
| } |
|
|
| $safeTensors = Join-Path $runDir "decoder.safetensors" |
| if (-not (Test-Path -LiteralPath $safeTensors)) { |
| & python (Join-Path $repo "tools\convert_decoder.py") ` |
| --training $training ` |
| --input $decoder ` |
| --output $safeTensors |
| if ($LASTEXITCODE -ne 0) { |
| throw "decoder SafeTensors conversion failed for seed $seed" |
| } |
| } |
| } |
|
|
| Write-Output "decoder matrix complete at $(Get-Date -Format o)" |
|
|
