File size: 1,444 Bytes
17e836b 96d6812 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | [*]
n-gpu-layers = 999
fit = off
cache-type-k = q8_0
cache-type-v = q8_0
repeat-penalty = 1.15
[Qwen3.8-6-Loop-9B-Heretic-Q4_K_S]
model = ./models/Qwen3.8-9B-Distill-uncensored-heretic.Q4_K_S.gguf
# Optional vision projector (kept in CPU RAM via mmproj-offload = false):
# mmproj = ./models/Qwen3.8-9B-Distill-uncensored-heretic.mmproj-f16.gguf
# mmproj-offload = false
lora = ./qwen3.8-9b-heretic-loop6-lora.gguf
override-kv = qwen35.num_loops=int:6,qwen35.recurrent_kv_cache_mode=str:per_loop
ctx-size = 8192
parallel = 1
cache-ram = 1024
ctx-checkpoints = 2
[Qwen3.8-4-Loop-9B-Heretic-Q4_K_S]
model = ./models/Qwen3.8-9B-Distill-uncensored-heretic.Q4_K_S.gguf
lora = ./qwen3.8-9b-heretic-loop6-lora.gguf
override-kv = qwen35.num_loops=int:4,qwen35.recurrent_kv_cache_mode=str:per_loop
ctx-size = 8192
parallel = 1
cache-ram = 1024
ctx-checkpoints = 2
[Qwen3.8-2-Loop-9B-Heretic-Q4_K_S]
model = ./models/Qwen3.8-9B-Distill-uncensored-heretic.Q4_K_S.gguf
lora = ./qwen3.8-9b-heretic-loop6-lora.gguf
override-kv = qwen35.num_loops=int:2,qwen35.recurrent_kv_cache_mode=str:per_loop
ctx-size = 8192
parallel = 1
cache-ram = 1024
ctx-checkpoints = 2
[Qwen3.8-1-Loop-9B-Heretic-Q4_K_S]
model = ./models/Qwen3.8-9B-Distill-uncensored-heretic.Q4_K_S.gguf
lora = ./qwen3.8-9b-heretic-loop6-lora.gguf
override-kv = qwen35.num_loops=int:1,qwen35.recurrent_kv_cache_mode=str:per_loop
ctx-size = 8192
parallel = 1
cache-ram = 1024
ctx-checkpoints = 2
|