Upload folder using huggingface_hub

scripts/compare_witgym_zero_shot.py

@@ -0,0 +1,110 @@
+"""WitGym vs zero-shot baseline — same model, same API backend."""
+import json
+import sys
+import time
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# 7 diverse scenarios (5 canonical + 2 extra)
+TEST_INPUTS = [
+    "I just got promoted to manager and I have no idea what I'm doing.",
+    "My coworker keeps stealing my lunch from the fridge.",
+    "I've been cc'd on an email chain I definitely should not be reading.",
+    "I'm pretending to understand cryptocurrency at dinner parties.",
+    "My therapist fell asleep during our session.",
+    "My boss called a quick sync that's been going for two hours.",
+    "I waved back at someone who wasn't waving at me.",
+]
+
+ZERO_SHOT_PROMPT = """You are a witty friend. The user shares an awkward or funny situation.
+Reply with ONE sharp, funny line. Maximum 2 sentences. No preamble.
+
+User: {user_input}
+
+Funny reply:"""
+
+
+def zero_shot_reply(user_input: str, model, tokenizer) -> str:
+    from witgym.model import generate_text
+
+    prompt = ZERO_SHOT_PROMPT.format(user_input=user_input)
+    raw = generate_text(prompt, model, tokenizer, config_type="generate")
+    return raw.strip().strip('"')
+
+
+def run_experiment(index_path: str = "data/index.npz") -> dict:
+    import os
+
+    os.environ.setdefault("LLM_BACKEND", "hf_api")
+
+    from witgym import config
+    from witgym.engine import WitGymEngine
+    from witgym.model import load_model
+
+    if not Path(index_path).exists():
+        print(f"[ERROR] Index missing: {index_path}")
+        sys.exit(1)
+
+    print(f"Model: {config.LLM_MODEL_ID} | Backend: {config.LLM_BACKEND}")
+    print(f"Providers: {config.HF_INFERENCE_PROVIDERS}\n")
+
+    # Shared embedder/index only — fresh engine per input (no conversation bleed)
+    shared = None
+    model, tokenizer = load_model()
+
+    rows = []
+    for i, user_input in enumerate(TEST_INPUTS, 1):
+        print(f"[{i}/{len(TEST_INPUTS)}] {user_input[:65]}...")
+
+        from witgym.engine import WitGymEngine, SharedResources
+
+        nonlocal_shared = shared
+        if nonlocal_shared is None:
+            nonlocal_shared = SharedResources(index_path=index_path)
+            shared = nonlocal_shared
+        engine = WitGymEngine(index_path=index_path, resources=nonlocal_shared)
+
+        t0 = time.time()
+        wg = engine.respond(user_input)
+        wg_elapsed = time.time() - t0
+
+        t0 = time.time()
+        zs = zero_shot_reply(user_input, model, tokenizer)
+        zs_elapsed = time.time() - t0
+
+        row = {
+            "input": user_input,
+            "witgym": {
+                "text": wg.selected,
+                "words": len(wg.selected.split()),
+                "latency_s": round(wg_elapsed, 1),
+                "archetype": wg.metadata.archetype.value,
+                "candidates": [c.text for c in wg.candidates],
+            },
+            "zero_shot": {
+                "text": zs,
+                "words": len(zs.split()),
+                "latency_s": round(zs_elapsed, 1),
+            },
+        }
+        rows.append(row)
+        print(f"  WitGym ({row['witgym']['words']}w, {wg_elapsed:.1f}s): {wg.selected}")
+        print(f"  Zero   ({row['zero_shot']['words']}w, {zs_elapsed:.1f}s): {zs}\n")
+
+    out = {
+        "model": config.LLM_MODEL_ID,
+        "backend": config.LLM_BACKEND,
+        "zero_shot_prompt": ZERO_SHOT_PROMPT,
+        "results": rows,
+    }
+    out_path = Path("data/eval_witgym_vs_zero_shot.json")
+    out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
+    print(f"Saved → {out_path}")
+    return out
+
+
+if __name__ == "__main__":
+    run_experiment()

witgym/config.py

@@ -23,7 +23,7 @@ EXTRACT_TEMP = 0.2
 EXTRACT_DO_SAMPLE = False
 GENERATE_TEMP = 1.0  # slightly lower to reduce arbitrary domain invention
 GENERATE_MIN_P = 0.06
-GENERATE_MAX_NEW_TOKENS = 40
+GENERATE_MAX_NEW_TOKENS = 72
 RANK_TEMP = 0.1
 EXTRACT_MAX_NEW_TOKENS = 220   # JSON never exceeds ~150 tokens; 220 = 47% safety margin
 

witgym/engine.py

@@ -314,8 +314,8 @@ class WitGymEngine:
             context_str = f"[Previous line — do NOT repeat]: {exclude_joke}\n" + context_str
         personas_to_run = ["cynic", "conviction", "absurdist"]
         if metadata.twist_potential <= 6:
-            personas_to_run = ["cynic", "conviction"]
-            logger.info(f"twist_potential={metadata.twist_potential} ≤ 6 — cynic + conviction (orthogonal pair)")
+            personas_to_run = ["cynic", "absurdist"]
+            logger.info(f"twist_potential={metadata.twist_potential} ≤ 6 — cynic + absurdist")
         elif metadata.twist_potential > 8:
             personas_to_run = ["cynic", "absurdist", "bisociate"]
             logger.info(f"twist_potential={metadata.twist_potential} > 8 — bisociate replaces conviction")

witgym/extractor.py

@@ -28,14 +28,15 @@ Return ONLY a JSON object with these exact fields (no explanation, no markdown,
 }}
 
 Think carefully about the ARCHETYPE — pick the one that most accurately describes the comedy mechanism hiding in this input.
-Archetype selection guidance (avoid overusing self_delusion):
-- status_assertion: claiming authority/status/rightness as if saying it makes it true
-- misplaced_conf: confident competence claim immediately unsupported by reality
-- anxiety_escalation: small trigger spun into catastrophe / inevitable doom logic
-- social_fail: awkward performance, norm violation, cringe, saying the wrong thing at the wrong time
-- power_inversion: low-status person is the only honest/correct one, or social power overrides institutional power
-- self_delusion: specifically a self-image story ("I'm fine / I'm great / I'm the best") contradicted by behavior/evidence in the same moment
-If unsure between self_delusion vs social_fail/anxiety_escalation, prefer the more specific one (social_fail/anxiety_escalation) unless there is an explicit self-image contradiction.
+Archetype selection guidance (avoid overusing social_fail — use it only for literal norm violations):
+- status_assertion: claiming authority/status/rightness as if saying it makes it true (e.g. "I just got promoted and have no idea what I'm doing" — claiming the title while admitting incompetence)
+- misplaced_conf: confident competence claim immediately unsupported by reality (e.g. "my boss called a quick sync that's been going for two hours" — the word "quick" is the mismatch)
+- anxiety_escalation: small trigger spun into catastrophe / inevitable doom logic (e.g. "my coworker keeps stealing my lunch" — escalating a minor loss into a power/control spiral)
+- social_fail: awkward performance, norm violation, cringe — ONLY when the person did/said something embarrassing in public (e.g. "I waved back at someone who wasn't waving at me" — literal public norm violation)
+- power_inversion: low-status person is the only honest/correct one, or the person with formal authority is visibly incompetent (e.g. "my therapist fell asleep during our session" — the helper needs help)
+- self_delusion: specifically a self-image story ("I'm fine / I'm great / I'm the best") contradicted by behavior/evidence (e.g. "I'm pretending to understand cryptocurrency at dinner parties" — actively constructing a false expert identity)
+CRITICAL: Do NOT use social_fail as a default. Most awkward situations are self_delusion, anxiety_escalation, power_inversion, or misplaced_conf. social_fail requires a public embarrassing action the person actually performed.
+If unsure between self_delusion vs social_fail, ask: did the person DO something publicly awkward (social_fail) or are they MAINTAINING a false self-image (self_delusion)?
 For twist_potential: score high if the input has self-delusion, status gap, or absurd logic. Score low if it is a neutral factual statement with no tension.
 If the input is ONLY a greeting, social opener, or typo-greeting with no described situation, set twist_potential to 1 and subtext to "greeting only — no situational content". Do not invent comedy tension.
 For connector: look for a single word or short phrase that carries an expected meaning in context AND a second meaning that reframes the situation. Most inputs will have null. Return null unless a genuine dual-reading exists (e.g. "manage" can mean control people or barely cope; "balance" can mean financial or emotional equilibrium).

witgym/router.py

@@ -83,14 +83,27 @@ def _parse_route(raw: str) -> Route | None:
 
 
 def classify_intent(text: str, model, tokenizer) -> Route:
-    """Route banter | quick_wit | coaching via LLM, with heuristic fallback."""
+    """Route banter | quick_wit | coaching via LLM, with heuristic fallback.
+
+    Heuristic takes priority when there is no explicit coaching signal — avoids
+    LLM misclassifying confessional 'I just X' situations as coaching.
+    """
     from witgym.model import generate_text
 
     if not text.strip():
         return "banter"
 
+    normalized = text.strip().lower()
+    has_coaching_signal = any(sig in normalized for sig in _COACHING_SIGNALS)
+
     heuristic = _heuristic_route(text)
 
+    # Skip LLM router when heuristic is confident and no explicit coaching signal:
+    # this prevents the LLM from misrouting confessional situation inputs to coaching.
+    if heuristic == "quick_wit" and not has_coaching_signal:
+        logger.info(f"Router: heuristic fast-path → 'quick_wit' for {text[:60]!r}")
+        return "quick_wit"
+
     raw = generate_text(
         ROUTER_PROMPT.format(user_input=text.strip()),
         model,
@@ -104,5 +117,5 @@ def classify_intent(text: str, model, tokenizer) -> Route:
         )
         route = heuristic
     else:
-        logger.info(f"Router: {route!r} for input {text[:60]!r}")
+        logger.info(f"Router: LLM → {route!r} for input {text[:60]!r}")
     return route