File size: 1,139 Bytes
a28e20f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | [CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string] $ServerExe,
[Parameter(Mandatory = $true)]
[string] $TargetModel,
[Parameter(Mandatory = $true)]
[string] $DraftModel,
[Parameter(Mandatory = $true)]
[string] $RocmBin,
[ValidateRange(1, 65535)]
[int] $Port = 8080
)
$ErrorActionPreference = "Stop"
foreach ($Path in @($ServerExe, $TargetModel, $DraftModel)) {
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
throw "File does not exist: $Path"
}
}
if (-not (Test-Path -LiteralPath $RocmBin -PathType Container)) {
throw "ROCm bin directory does not exist: $RocmBin"
}
$env:PATH = "$(Resolve-Path -LiteralPath $RocmBin);$env:PATH"
& $ServerExe `
-m $TargetModel `
-md $DraftModel `
--host 127.0.0.1 `
--port $Port `
--alias strix-halo-dflash2 `
--ctx-size 32768 `
--batch-size 2048 `
--ubatch-size 512 `
-ngl 99 `
--spec-draft-ngl 99 `
--spec-type draft-dflash `
--spec-draft-n-max 7 `
--spec-draft-p-min 0.90 `
--parallel 1 `
--metrics `
--cache-type-k q8_0 `
--cache-type-v q8_0
|