Update services/kb_creation.py

services/kb_creation.py

@@ -71,26 +71,24 @@ def _split_by_sections(doc: Document) -> List[Tuple[str, List[str]]]:
 def _paragraphs_to_lines(paragraphs: List[str]) -> List[str]:
     """
     Split paragraphs into bullet-aware lines:
-    - Preserve bullets and numbered list lines as separate atomic lines.
-    - Split long paragraphs by '. ' into manageable lines.
+    - Preserve bullets/numbered list lines as separate atomic lines.
+    - Split long paragraphs by sentence boundaries.
     """
     lines: List[str] = []
     for p in (paragraphs or []):
         p = (p or "").strip()
         if not p:
             continue
-        # If looks like a bullet/numbered item, keep as is
         if BULLET_RE.match(p):
             lines.append(p)
             continue
-        # Otherwise split by sentence boundaries
         parts = [s.strip() for s in re.split(r"(?<=[.!?])\s+", p) if s.strip()]
         lines.extend(parts)
     return lines
 
-def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: List[str], max_words: int = 300) -> List[str]:
+def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: List[str], max_words: int = 160) -> List[str]:
     """
-    Smaller chunks for better recall; bullet-aware.
+    Smaller chunks (≈160 words), bullet-aware for better recall of error bullets.
     """
     lines = _paragraphs_to_lines(paragraphs)
     chunks: List[str] = []
@@ -100,7 +98,6 @@ def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: Lis
     for ln in lines:
         w = ln.split()
         if current_len + len(w) > max_words or (BULLET_RE.match(ln) and current):
-            # close current chunk
             chunk = " ".join(current).strip()
             if chunk:
                 chunks.append(chunk)
@@ -220,7 +217,7 @@ def ingest_documents(folder_path: str) -> None:
         total_chunks = 0
 
         for s_idx, (section_title, paras) in enumerate(sections):
-            chunks = _chunk_text_with_context(doc_title, section_title, paras, max_words=300)
+            chunks = _chunk_text_with_context(doc_title, section_title, paras, max_words=160)
             total_chunks += len(chunks)
 
             base_intent = _infer_intent_tag(section_title)
@@ -255,7 +252,6 @@ def ingest_documents(folder_path: str) -> None:
                     except Exception as e2:
                         print(f"[KB] ERROR: Upsert failed for {doc_id}: {e2}")
 
-                # Build BM25 index entries
                 tokens = _tokenize(chunk)
                 tf: Dict[str, int] = {}
                 for tkn in tokens:
@@ -395,7 +391,7 @@ def search_knowledge_base(query: str, top_k: int = 10) -> dict:
         "ids": ids,
     }
 
-# ---------------------------- Hybrid search (improved) ----------------------------
+# ---------------------------- Hybrid search (improved + exact-match rerank) ----------------------------
 ACTION_SYNONYMS = {
     "create": ["create", "creation", "add", "new", "generate"],
     "update": ["update", "modify", "change", "edit"],
@@ -481,8 +477,12 @@ def _intent_weight(meta: dict, user_intent: str) -> float:
     st = ((meta or {}).get("section", "") or "").lower()
     topics = (meta or {}).get("topic_tags", "") or ""
     topic_list = [t.strip() for t in topics.split(",") if t.strip()]
-    if user_intent == "errors" and (any(k in st for k in ["escalation", "permissions", "access", "known issues", "common issues"]) or ("permissions" in topic_list)):
-        return 0.95
+    # Prefer errors sections strongly
+    if user_intent == "errors" and (
+        any(k in st for k in ["common errors", "known issues", "common issues", "errors", "escalation", "permissions", "access"])
+        or ("permissions" in topic_list)
+    ):
+        return 1.10  # stronger than before
     if user_intent == "steps" and any(k in st for k in ["inbound receiving", "receiving", "goods receipt", "grn"]):
         return 0.75
     return -0.2
@@ -512,30 +512,26 @@ def _phrase_boost_score(text: str, q_terms: List[str]) -> float:
     score = 0.0
     for bg in bigrams:
         if bg and bg in low:
-            score += 0.35
+            score += 0.40
     for tg in trigrams:
         if tg and tg in low:
-            score += 0.60
-    return min(score, 1.5)
+            score += 0.70
+    return min(score, 2.0)
 
 def _literal_query_match_boost(text: str, query_norm: str) -> float:
-    """
-    Extra boost if the exact normalized query substring (or key tokens) appear in the chunk.
-    Helps errors like 'item mismatch' pick the right KB line.
-    """
+    """Extra boost if exact normalized query substring or bigrams appear."""
     t = (text or "").lower()
     q = (query_norm or "").lower()
     boost = 0.0
     if q and q in t:
-        boost += 0.6
-    # Also check key 2-word error tokens present in query (e.g., 'item mismatch')
+        boost += 0.8
     toks = [tok for tok in q.split() if len(tok) > 2]
     bigrams = _make_ngrams(toks, 2)
     for bg in bigrams:
         if bg in t:
-            boost += 0.6
+            boost += 0.8
             break
-    return min(boost, 1.2)
+    return min(boost, 1.6)
 
 def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6, beta: float = 0.4) -> dict:
     norm_query = _normalize_query(query)
@@ -544,7 +540,7 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
     actions = _extract_actions(query)
     user_modules = _extract_modules_from_query(query)
 
-    sem_res = search_knowledge_base(norm_query, top_k=max(top_k, 30))
+    sem_res = search_knowledge_base(norm_query, top_k=max(top_k, 40))
     sem_docs = sem_res.get("documents", [])
     sem_metas = sem_res.get("metadatas", [])
     sem_dists = sem_res.get("distances", [])
@@ -560,7 +556,7 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
 
     sem_sims = [dist_to_sim(d) for d in sem_dists]
 
-    bm25_hits = bm25_search(norm_query, top_k=max(50, top_k * 5))
+    bm25_hits = bm25_search(norm_query, top_k=max(80, top_k * 6))
     bm25_max = max([s for _, s in bm25_hits], default=1.0)
     bm25_norm_pairs = [(idx, (score / bm25_max) if bm25_max > 0 else 0.0) for idx, score in bm25_hits]
     bm25_id_to_norm, bm25_id_to_text, bm25_id_to_meta = {}, {}, {}
@@ -573,12 +569,12 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
     union_ids = set(sem_ids) | set(bm25_id_to_norm.keys())
 
     gamma = 0.30   # meta overlap
-    delta = 0.50   # intent boost
+    delta = 0.55   # intent boost (stronger)
     epsilon = 0.30 # action weight
     zeta = 0.65    # module weight
-    eta = 0.45     # phrase-level boost
+    eta = 0.50     # phrase-level boost (stronger)
     theta = 0.40   # heading alignment bonus
-    iota = 0.40    # literal query match boost
+    iota = 0.60    # literal query match boost (stronger)
 
     combined_records_ext: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]] = []
     for cid in union_ids:
@@ -631,6 +627,24 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             (cid, final_score, (sem_dist if sem_dist is not None else 999.0), text, meta, m_overlap, intent_boost, act_wt, mod_wt, phrase_wt, heading_bonus, literal_wt)
         )
 
+    # ---- Exact-match rerank for errors ----
+    if user_intent == "errors":
+        exact_hits = []
+        for rec in combined_records_ext:
+            text_lower = (rec[3] or "").lower()
+            if any(phrase in text_lower for phrase in [
+                norm_query,  # whole normalized query
+                # common 2-gram patterns extracted from the query
+                *(_make_ngrams([tok for tok in norm_query.split() if len(tok) > 2], 2))
+            ]):
+                exact_hits.append(rec)
+        if exact_hits:
+            # Move exact hits to front and keep order by current final_score
+            rest = [r for r in combined_records_ext if r not in exact_hits]
+            exact_hits.sort(key=lambda x: x[1], reverse=True)
+            rest.sort(key=lambda x: x[1], reverse=True)
+            combined_records_ext = exact_hits + rest
+
     from collections import defaultdict
     doc_groups: Dict[str, List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]]] = defaultdict(list)
     for rec in combined_records_ext:
@@ -648,8 +662,11 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         total_heading = sum(r[10] for r in recs)
         total_literal = sum(r[11] for r in recs)
         total_penalty = sum(min(0.0, r[6]) for r in recs) + sum(min(0.0, r[7]) for r in recs)
-        esc_weight = 0.3 if any("escalation" in ((r[4] or {}).get("section", "")).lower() for r in recs) else 0.0
-        perm_weight = 0.3 if any("permissions" in (((r[4] or {}).get("topic_tags") or [])) for r in recs) else 0.0
+        # Errors doc prior: if many chunks are from an errors/known issues section, add a bonus
+        errors_section_bonus = 0.0
+        if any("error" in ((r[4] or {}).get("section", "")).lower() or "known issues" in ((r[4] or {}).get("section", "")).lower()
+               or "common issues" in ((r[4] or {}).get("section", "")).lower() for r in recs):
+            errors_section_bonus = 0.5
         return (
             total_score
             + 0.4 * total_overlap
@@ -658,9 +675,9 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             + 0.8 * total_module
             + 0.6 * total_phrase
             + 0.6 * total_heading
-            + 0.6 * total_literal
+            + 0.7 * total_literal
+            + errors_section_bonus
             + 0.3 * total_penalty
-            + esc_weight + perm_weight
         )
 
     best_doc, best_doc_prior = None, -1.0
@@ -734,6 +751,7 @@ def get_best_errors_section_text(filename: str) -> str:
             or "access" in sec
             or "known issues" in sec
             or "common issues" in sec
+            or "errors" in sec
             or ("permissions" in topic_list)
         ):
             t = (d.get("text") or "").strip()