fix(core): resolve caching, lazy load, and thread leak defects

- Prevent generic fallback response from caching and poisoning state.
- Read LOCAL_7B_P95 from environment variables instead of a magic hardcoded number.
- Refactor local tier to stream generation and abort executor thread if deadline is exceeded.
- Remove over-broad words 'JSON' and 'system' from default taboo list.
- Wrap local model loading in a lazy-load function to prevent HF space boot failure.

scripts/composer.py

@@ -64,7 +64,7 @@ async def compose_with_fallback(prompt: str, full_system: str, local_system: str
                     return await func(p, system_to_use, timeout)
                 else:
                     try:
-                        return await asyncio.wait_for(func(p, system_to_use), timeout=deadline - time.time())
+                        return await asyncio.wait_for(func(p, system_to_use, deadline), timeout=deadline - time.time())
                     except asyncio.TimeoutError:
                         return None
                         
@@ -140,7 +140,7 @@ async def compose(category: dict, merchant: dict, trigger: dict, customer: Optio
     # 2. Dynamic Tone & Taboos
     voice = category.get("voice", {})
     tone = voice.get("tone", "professional peer")
-    taboos_list = voice.get("taboos", ["internal metrics", "JSON", "system", "guaranteed"])
+    taboos_list = voice.get("taboos", ["guaranteed", "100%", "click here", "promise"])
     taboos = ", ".join(taboos_list)
     
     prefix = "Dr. " if category.get('slug') == 'dentists' else ""
@@ -226,5 +226,6 @@ Example Output format for a good message:
     
     # Add sender attribution required by the challenge
     res["send_as"] = "merchant_on_behalf" if customer else "vera"
-    _cache[key] = res
+    if res.get("rationale") != "fallback_default":
+        _cache[key] = res
     return res

scripts/llm.py

@@ -13,7 +13,8 @@ load_dotenv()
 logger = logging.getLogger("VeraArchitect")
 
 LOCAL_MAX_TOKENS = 250
-local_model = None
+_local_model_instance = None
+_local_model_loaded = False
 
 try:
     _schema = {"type":"object","properties":{"body":{"type":"string"},"cta":{"type":"string"},"rationale":{"type":"string"}},"required":["body","cta","rationale"]}
@@ -21,23 +22,31 @@ try:
 except Exception:
     JSON_GRAMMAR = None
 
-try:
-    LOCAL_7B_P95 = 37.47 
-    if LOCAL_7B_P95 <= 14.0:
-        local_repo = "bartowski/Qwen2.5-7B-Instruct-GGUF"
-        local_file = "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
-        LOCAL_MAX_TOKENS = 150
-        logger.info(f"Using 7b for local tier since p95 latency ({LOCAL_7B_P95}s) is within 14s budget.")
-    else:
-        local_repo = "Qwen/Qwen2.5-3B-Instruct-GGUF"
-        local_file = "qwen2.5-3b-instruct-q4_k_m.gguf"
-        LOCAL_MAX_TOKENS = 250
-        logger.info(f"7b p95 {LOCAL_7B_P95}s exceeds 14s budget, using 3b instead.")
+def get_local_model():
+    global _local_model_instance, _local_model_loaded, LOCAL_MAX_TOKENS
+    if _local_model_loaded:
+        return _local_model_instance
+    
+    _local_model_loaded = True
+    try:
+        LOCAL_7B_P95 = float(os.getenv("LOCAL_7B_P95", "999.0")) 
+        if LOCAL_7B_P95 <= 14.0:
+            local_repo = "bartowski/Qwen2.5-7B-Instruct-GGUF"
+            local_file = "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
+            LOCAL_MAX_TOKENS = 150
+            logger.info(f"Using 7b for local tier since p95 latency ({LOCAL_7B_P95}s) is within 14s budget.")
+        else:
+            local_repo = "Qwen/Qwen2.5-3B-Instruct-GGUF"
+            local_file = "qwen2.5-3b-instruct-q4_k_m.gguf"
+            LOCAL_MAX_TOKENS = 250
+            logger.info(f"7b p95 {LOCAL_7B_P95}s exceeds 14s budget, using 3b instead.")
+            
+        _local_model_instance = Llama.from_pretrained(repo_id=local_repo, filename=local_file, n_ctx=2048, n_threads=os.cpu_count(), verbose=False)
+    except Exception as e:
+        logger.warning(f"Failed to load local model: {e}")
+        _local_model_instance = None
         
-    local_model = Llama.from_pretrained(repo_id=local_repo, filename=local_file, n_ctx=2048, n_threads=os.cpu_count(), verbose=False)
-except Exception as e:
-    logger.warning(f"Failed to load local model: {e}")
-    local_model = None
+    return _local_model_instance
 
 def clean_json(text: str) -> Optional[dict]:
     try:
@@ -88,21 +97,37 @@ async def call_gemini(prompt: str, system: str, timeout: float) -> Optional[dict
         res.raise_for_status()
         return clean_json(res.json()["candidates"][0]["content"]["parts"][0]["text"])
 
-async def call_local(prompt: str, system: str) -> Optional[dict]:
-    if not local_model: return None
+async def call_local(prompt: str, system: str, deadline: float = None) -> Optional[dict]:
+    model = get_local_model()
+    if not model: return None
+    
     def run_inference():
         kwargs = {
             "messages": [{"role": "system", "content": system + "\n\nRETURN JSON ONLY."}, {"role": "user", "content": prompt}],
             "temperature": 0.2,
             "max_tokens": LOCAL_MAX_TOKENS,
+            "stream": True,
         }
         if JSON_GRAMMAR:
             kwargs["grammar"] = JSON_GRAMMAR
         else:
             kwargs["response_format"] = {"type": "json_object"}
             
-        res = local_model.create_chat_completion(**kwargs)
-        return res["choices"][0]["message"]["content"]
+        res_text = ""
+        try:
+            for chunk in model.create_chat_completion(**kwargs):
+                if deadline and time.time() > deadline:
+                    logger.warning("Local tier exceeded deadline mid-generation. Aborting thread.")
+                    return None
+                delta = chunk["choices"][0].get("delta", {})
+                if "content" in delta and delta["content"]:
+                    res_text += delta["content"]
+        except Exception as e:
+            logger.warning(f"Local stream aborted: {e}")
+            return None
+        return res_text
+        
     loop = asyncio.get_event_loop()
     text = await loop.run_in_executor(None, run_inference)
+    if not text: return None
     return clean_json(text)