File size: 2,285 Bytes
d79a813 11b8dc8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | ---
base_model:
- unsloth/Qwen3.8-27B-GGUF
---
Just plain `llama-quantize`d version of Unsloth's GGUFs :)
My `llama-swap` conf for `Q4_0_ROCMFP4_COHERENT`:
```
"qwen38":
name: "qwen38"
env:
- "MODEL_ID=qwen38"
filters:
stripParams: "temperature, top_k, top_p, repeat_penalty, min_p, presence_penalty"
setParamsByID:
"${MODEL_ID}:low-reasoning":
chat_template_kwargs:
reasoning_effort: low
"${MODEL_ID}:med-reasoning":
chat_template_kwargs:
reasoning_effort: medium
"${MODEL_ID}:instruct-nothink":
temperature: 0.7
top_p: 0.8
min_p: 0.0
presence_penalty: 1.5
repeat_penalty: 1.0
chat_template_kwargs:
enable_thinking: false
preserve_thinking: false
"${MODEL_ID}:reasoning-nothink":
temperature: 0.85
chat_template_kwargs:
enable_thinking: false
preserve_thinking: false
proxy: http://host.docker.internal:${PORT}
cmdStop: docker stop rocmfpx-docker-rocm-llama-${PORT}
cmd: |
${rocmfpx-docker-rocm-llama}
--model /mnt/models/Qwen3.8-27B-ROCMFPX/Qwen3.8-27B-Q4_0_ROCMFP4_COHERENT.gguf
--ctx-size 262144
--ctx-checkpoints 32
--checkpoint-every-n-tokens 8192
--seed 69420
--temperature 1.0
--top-p 0.95
--top-k 20
--min-p 0.0
--presence_penalty 0.0
--repeat_penalty 1.0
--spec-draft-type-k q4_0
--spec-draft-type-v q4_0
--cache-type-k f16
--cache-type-v f16
--spec-type draft-mtp
--spec-draft-ngl all
--spec-draft-n-max 5
--spec-draft-threads 16
--spec-draft-threads-batch 32
--spec-draft-poll 1
--spec-draft-poll-batch 1
--chat-template-kwargs '{"preserve_thinking": true}'
``` |