promote hybrid: 8B map + 30B read + split build (VRAM) + FLUX preload + CUDA cache
Browse files- arcana/llm.py +4 -3
arcana/llm.py
CHANGED
|
@@ -150,9 +150,10 @@ class LocalLLM:
|
|
| 150 |
timeout: float | None = None) -> str:
|
| 151 |
from .gpu_models import llama_map, llama_read
|
| 152 |
if json_mode:
|
| 153 |
-
#
|
| 154 |
-
|
| 155 |
-
|
|
|
|
| 156 |
return llama_read(system, user, max_tokens=self.max_tokens,
|
| 157 |
temperature=self.temperature)
|
| 158 |
|
|
|
|
| 150 |
timeout: float | None = None) -> str:
|
| 151 |
from .gpu_models import llama_map, llama_read
|
| 152 |
if json_mode:
|
| 153 |
+
# mapping → 30B, think-suppressed (fast); generous tokens so the full
|
| 154 |
+
# 22-card JSON never truncates
|
| 155 |
+
return llama_map(system, user, max_tokens=12000, temperature=0.5)
|
| 156 |
+
# prose readings → 30B (think-suppressed for clean, fast output)
|
| 157 |
return llama_read(system, user, max_tokens=self.max_tokens,
|
| 158 |
temperature=self.temperature)
|
| 159 |
|