Update main.py

main.py

@@ -216,6 +216,61 @@ def _get_steps_by_title_keywords_global(keywords: List[str], prefer_doc: Optiona
     return best_doc_g, best_sec_g, best_text_g
 
 
+# --- Title or text global selector (ultimate fallback) ---
+def _get_steps_by_title_or_text_global(keywords: List[str], prefer_doc: Optional[str] = None) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+    """
+    Search ALL SOP docs for a 'steps' section where either the section title OR any step text
+    contains any of `keywords`. Returns (doc_name, section_title, steps_text).
+    """
+    if not keywords:
+        return None, None, None
+    keys_low = [k.strip().lower() for k in keywords if k and k.strip()]
+
+    # (score, doc, sec, text)
+    candidates: List[Tuple[float, str, str, str]] = []
+    seen = set()
+
+    for d in bm25_docs:
+        m = d.get("meta", {}) or {}
+        if m.get("intent_tag") != "steps":
+            continue
+        doc = m.get("filename")
+        sec = (m.get("section") or "").strip()
+        txt = (d.get("text") or "").strip()
+        if not doc or not sec or not txt:
+            continue
+        key_hit = False
+        sec_low = sec.lower()
+        txt_low = txt.lower()
+        if any(k in sec_low for k in keys_low) or any(k in txt_low for k in keys_low):
+            key_hit = True
+        if not key_hit:
+            continue
+
+        # aggregate once per (doc, sec)
+        if (doc, sec) in seen:
+            continue
+        full = get_section_text(doc, sec)
+        if not full or not full.strip():
+            continue
+
+        score = 1.0
+        if prefer_doc and doc == prefer_doc:
+            score += 0.4
+        mtags = (m.get("module_tags") or "").lower()
+        if "appointments" in mtags:
+            score += 0.3
+        candidates.append((score, doc, sec, full.strip()))
+        seen.add((doc, sec))
+
+    if not candidates:
+        return None, None, None
+
+    candidates.sort(key=lambda x: x[0], reverse=True)
+    _, bdoc, bsec, btext = candidates[0]
+    return bdoc, bsec, btext
+
+
 # --- Action-targeted steps selector (uses existing KB metadata) ---
 def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
     """
@@ -227,7 +282,6 @@ def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
     if not best_doc or not actions:
         return None
     act_set = set(a.strip().lower() for a in actions if a)
-    # Collect candidate sections in this doc that are 'steps' and have an action_tag we need
     candidates = []
     for d in bm25_docs:
         m = d.get("meta", {})
@@ -235,7 +289,6 @@ def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
             tag = (m.get("action_tag") or "").strip().lower()
             if tag and tag in act_set:
                 candidates.append(m.get("section"))
-    # Prefer the first matched section with non-empty text
     for title in candidates:
         txt = get_section_text(best_doc, title)
         if txt and txt.strip():
@@ -402,11 +455,6 @@ def _norm_text(s: str) -> str:
 
 
 def _split_sop_into_steps(numbered_text: str) -> list:
-    """
-    Split a numbered/bulleted SOP block (already passed through _ensure_numbering)
-    into atomic steps. Returns a list of raw step strings (order preserved).
-    Safe for circled digits, '1.' styles, and bullets.
-    """
     lines = [ln.strip() for ln in (numbered_text or "").splitlines() if ln.strip()]
     steps = []
     for ln in lines:
@@ -417,7 +465,6 @@ def _split_sop_into_steps(numbered_text: str) -> list:
 
 
 def _soft_match_score(a: str, b: str) -> float:
-    # Simple Jaccard-like score on tokens for fuzzy matching
     ta = set(_norm_text(a).split())
     tb = set(_norm_text(b).split())
     if not ta or not tb:
@@ -437,38 +484,27 @@ def _detect_next_intent(user_query: str) -> bool:
 
 
 def _resolve_next_steps(user_query: str, numbered_text: str, max_next: int = 6, min_score: float = 0.35):
-    """
-    If 'what's next' intent is detected and we can reliably match the user's
-    referenced line to a SOP step, return ONLY the subsequent steps.
-    Else return None to preserve current behavior.
-    """
     if not _detect_next_intent(user_query):
         return None
-
     steps = _split_sop_into_steps(numbered_text)
     if not steps:
         return None
-
     q = user_query or ""
     best_idx, best_score = -1, -1.0
     for idx, step in enumerate(steps):
         score = 1.0 if _norm_text(step) in _norm_text(q) else _soft_match_score(q, step)
         if score > best_score:
             best_score, best_idx = score, idx
-
     if best_idx < 0 or best_score < min_score:
-        return None  # fallback to full SOP
-
+        return None
     start = best_idx + 1
     if start >= len(steps):
-        return []  # user is at final step
-
+        return []
     end = min(start + max_next, len(steps))
     return steps[start:end]
 
 
 def _format_steps_as_numbered(steps: list) -> str:
-    """Render a small list of steps with circled numbers for visual continuity."""
     circled = {
         1: "\u2460", 2: "\u2461", 3: "\u2462", 4: "\u2463", 5: "\u2464",
         6: "\u2465", 7: "\u2466", 8: "\u2467", 9: "\u2468", 10: "\u2469",
@@ -482,17 +518,6 @@ def _format_steps_as_numbered(steps: list) -> str:
 
 
 def _filter_error_lines_by_query(text: str, query: str, max_lines: int = 1) -> str:
-    """
-    Pick the most relevant 'Common Errors & Resolution' bullet(s) for the user's message.
-    Generic (SOP-agnostic) scoring:
-      1) error-family match (NOT_FOUND/MISMATCH/LOCKED/PERMISSION/TIMEOUT/SYNC),
-      2) anchored starts (line begins with error heading),
-      3) multi-word overlap (bigrams/trigrams),
-      4) token overlap,
-      5) formatting bonus for bullets/headings.
-
-    Returns exactly `max_lines` best-scoring lines (defaults to 1).
-    """
     def _norm(s: str) -> str:
         s = (s or "").lower()
         s = re.sub(r"[^\w\s]", " ", s)
@@ -546,7 +571,7 @@ def _filter_error_lines_by_query(text: str, query: str, max_lines: int = 1) -> s
             0.30 * token_overlap
         )
 
-        if re.match(r"^\s*[-*\u2022]\s*", ln):  # bullet
+        if re.match(r"^\s*[-*\u2022]\s*", ln):
             score += 0.10
         heading = ln_norm.split(":")[0].strip()
         if heading and (heading in q or (first2 and first2 in heading)):
@@ -617,7 +642,6 @@ def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
         any(w in msg_norm for w in ("ticket", "incident", "servicenow", "snow")) or
         bool(re.search(r"\binc\d{5,}\b", msg_norm, flags=re.IGNORECASE))
     )
-    # Disambiguation: if it's a domain status query and not clearly ticket/incident, do NOT route to ticket-status.
     if (not base_has_status) or (base_has_status and not has_ticket_marker and _is_domain_status_context(msg_norm)):
         return {}
     patterns = [
@@ -693,11 +717,6 @@ def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str,
 
 
 def _extract_errors_only(text: str, max_lines: int = 12) -> str:
-    """
-    Collect error bullets/heading-style lines from the SOP errors section.
-    Generic: keeps bullet points (•, -, *), and lines that look like "Heading: details".
-    This ensures items like 'ASN not found', 'Trailer status locked', etc., are preserved.
-    """
     kept: List[str] = []
     for ln in _normalize_lines(text):
         if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln):
@@ -768,7 +787,7 @@ def _classify_resolution_llm(user_message: str) -> bool:
 Return only 'true' or 'false'.
 Message: {user_message}"""
     headers = {"Content-Type": "application/json"}
-    payload = {"contents": [{"parts": [{"text": prompt}]}]}
+    payload = {"contents": [{"parts": [{"text": prompt}]}]]}
     try:
         resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=12, verify=GEMINI_SSL_VERIFY)
         data = resp.json()
@@ -846,7 +865,7 @@ def _set_incident_resolved(sys_id: str) -> bool:
         notes_field = os.getenv("SERVICENOW_RESOLUTION_NOTES_FIELD", "close_notes")
         payload_C = clean({
             "state": "6",
-            code_field: close_notes_val,  # (if you have custom fields, adjust here)
+            code_field: close_notes_val,
             notes_field: close_notes_val,
             "caller_id": caller_sysid,
             "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
@@ -866,10 +885,6 @@ def _set_incident_resolved(sys_id: str) -> bool:
 
 # ------------------------------ Prereq helper ------------------------------
 def _find_prereq_section_text(best_doc: str) -> str:
-    """
-    Return the prerequisites section text, trying common heading variants.
-    Generic for future SOPs—no document-specific keywords.
-    """
     variants = [
         "Pre-Requisites",
         "Prerequisites",
@@ -1044,7 +1059,7 @@ async def chat_with_ai(input_data: ChatInput):
             except Exception as e:
                 raise HTTPException(status_code=500, detail=safe_str(e))
 
-        # Hybrid KB search
+        # Hybrid KB search (keep raw; filter only used for gating)
         kb_results = hybrid_search_knowledge_base(input_data.user_message, top_k=10, alpha=0.6, beta=0.4)
         documents = kb_results.get("documents", [])
         metadatas = kb_results.get("metadatas", [])
@@ -1068,11 +1083,10 @@ async def chat_with_ai(input_data: ChatInput):
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
 
-        # Compute filter info for gating only; do NOT use the filtered text for steps
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
         filtered_context = filtered_text
 
-        context = context_raw  # keep raw; we'll decide below
+        context = context_raw
         context_found = bool(context.strip())
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
@@ -1110,7 +1124,7 @@ async def chat_with_ai(input_data: ChatInput):
         if detected_intent == "neutral" and any(h in sec_title for h in PREREQ_HEADINGS):
             detected_intent = "prereqs"
 
-        # --- Module-aware steps nudge (appointments, picking, shipping, etc.) ---
+        # --- Module-aware steps nudge ---
         STEPS_TERMS = ("how to", "procedure", "perform", "steps", "do", "navigate")
         ACTIONS_PRESENT = any(s in msg_low for syns in ACTION_SYNONYMS_EXT.values() for s in syns)
         mod_tags = ((top_meta or {}).get("module_tags") or "").lower()
@@ -1194,97 +1208,6 @@ async def chat_with_ai(input_data: ChatInput):
                     return act
             return None
 
-        def _strip_boilerplate(raw_context: str) -> str:
-            """Remove document title/date/author/change-history noise from steps."""
-            MONTH_TERMS = ("january", "february", "march", "april", "may", "june",
-                           "july", "august", "september", "october", "november", "december")
-            lines = _normalize_lines(raw_context)
-            cleaned: List[str] = []
-            for ln in lines:
-                low = ln.lower()
-                is_change_hist = ("change history" in low) or ("initial draft" in low) or ("review" in low) or ("version" in low)
-                has_month_year = any(m in low for m in MONTH_TERMS) and bool(re.search(r"\b20\d{2}\b", low))
-                is_title_line = ("sop document" in low) or ("contents" in low)
-                if is_change_hist or has_month_year or is_title_line:
-                    continue
-                cleaned.append(ln)
-            return "\n".join(cleaned).strip()
-
-        def _extract_action_block(raw_context: str, target_act: Optional[str]) -> str:
-            """
-            Extract the contiguous block of lines for the target action (create/update/delete).
-            Start when a line mentions the target action OR looks procedural for 'create',
-            and stop ONLY when a line is a clear boundary:
-              - inline heading for another topic (e.g., 'Appointment schedule updation', 'Appointment deletion'), OR
-              - a line that strongly signals a different action (update/delete) via extended synonyms.
-            """
-            if not raw_context.strip():
-                return raw_context
-
-            lines = _normalize_lines(raw_context)
-            if not lines or not target_act:
-                return raw_context
-
-            INLINE_BOUNDARIES = (
-                "appointment schedule updation",
-                "schedule updation",
-                "appointment deletion",
-                "deletion",
-            )
-
-            other_terms: List[str] = []
-            for act, syns in ACTION_SYNONYMS_EXT.items():
-                if act != target_act:
-                    other_terms.extend(syns)
-            other_terms_low = set(t.lower() for t in other_terms)
-
-            def is_boundary_line(low: str) -> bool:
-                if any(h in low for h in INLINE_BOUNDARIES):
-                    return True
-                if any(t in low for t in other_terms_low):
-                    return True
-                return False
-
-            PROCEDURAL_VERBS = ("select", "choose", "click", "open", "add", "assign", "save",
-                                "navigate", "tag", "displayed", "triggered")
-            def is_procedural(low: str) -> bool:
-                return any(v in low for v in PROCEDURAL_VERBS)
-
-            target_terms_low = set(t.lower() for t in ACTION_SYNONYMS_EXT.get(target_act, []))
-
-            started = False
-            block: List[str] = []
-
-            for ln in lines:
-                low = ln.lower()
-
-                contains_target = any(t in low for t in target_terms_low)
-                if not started:
-                    if contains_target or (target_act == "create" and is_procedural(low)):
-                        started = True
-                        block.append(ln)
-                    continue
-
-                if is_boundary_line(low):
-                    break
-
-                block.append(ln)
-
-            return "\n".join(block).strip() if block else raw_context
-
-        def _filter_steps_by_action(raw_context: str, asked_act: Optional[str]) -> str:
-            cleaned = _strip_boilerplate(raw_context)
-            block = _extract_action_block(cleaned, asked_act)
-            if asked_act:
-                other_terms: List[str] = []
-                for act, syns in ACTION_SYNONYMS_EXT.items():
-                    if act != asked_act:
-                        other_terms.extend(syns)
-                lines = _normalize_lines(block)
-                lines = [ln for ln in lines if not any(t in ln.lower() for t in other_terms)]
-                block = "\n".join(lines).strip() if lines else block
-            return block
-
         escalation_line = None
         full_errors = None
         next_step_applied = False
@@ -1315,7 +1238,7 @@ async def chat_with_ai(input_data: ChatInput):
                         best_doc = alt_doc2
                         full_steps = alt_steps2
 
-                # 1c) Title-keywords global fallback (robust when action_tag is missing)
+                # 1c) Title keywords global fallback
                 if not full_steps:
                     if asked_action == "update":
                         kd = ["updation", "update", "reschedule"]
@@ -1330,7 +1253,16 @@ async def chat_with_ai(input_data: ChatInput):
                         best_doc = alt_doc3
                         full_steps = alt_steps3
 
-                # Existing local fallbacks
+                # 1d) Title OR text global fallback (ultimate)
+                if not full_steps:
+                    alt_doc4, alt_sec4, alt_steps4 = _get_steps_by_title_or_text_global(
+                        ["updation", "update", "deletion", "delete", "cancel", "creation"], prefer_doc=best_doc
+                    )
+                    if alt_steps4:
+                        best_doc = alt_doc4
+                        full_steps = alt_steps4
+
+                # Existing local fallbacks (doc-scoped)
                 if not full_steps:
                     default_sec = _pick_default_action_section(best_doc)
                     if default_sec:
@@ -1350,11 +1282,8 @@ async def chat_with_ai(input_data: ChatInput):
                     if not any(s in low_steps for s in SAVE_SYNS):
                         full_steps = (full_steps or "").rstrip() + "\n" + save_lines
 
-                # Remove aggressive trimming; keep full section
-                asked_action = _detect_action_from_query(input_data.user_message)
-                full_steps = _filter_steps_by_action(full_steps, asked_action)
-
-                # Number steps and apply next-step logic
+                # IMPORTANT: Do NOT trim steps by action — return full section
+                # (We keep "what's next" behavior only.)
                 numbered_full = _ensure_numbering(full_steps)
                 next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
 
@@ -1373,7 +1302,6 @@ async def chat_with_ai(input_data: ChatInput):
                     context = full_steps
                     context_preformatted = False
 
-                # Clear filter info for debug clarity
                 filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
                 context_found = True
 
@@ -1399,8 +1327,6 @@ async def chat_with_ai(input_data: ChatInput):
             else:
                 full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
                 if full_steps:
-                    asked_action = _detect_action_from_query(input_data.user_message)
-                    full_steps = _filter_steps_by_action(full_steps, asked_action)
                     context = full_steps
                     detected_intent = "steps"
                     context_preformatted = False
@@ -1412,14 +1338,11 @@ async def chat_with_ai(input_data: ChatInput):
             else:
                 full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
                 if full_steps:
-                    asked_action = _detect_action_from_query(input_data.user_message)
-                    full_steps = _filter_steps_by_action(full_steps, asked_action)
                     context = full_steps
                     detected_intent = "steps"
                     context_preformatted = False
 
         else:
-            # Neutral or other intents: use filtered context
             context = filtered_context
 
         language_hint = _detect_language_hint(input_data.user_message)
@@ -1436,7 +1359,7 @@ async def chat_with_ai(input_data: ChatInput):
 ### Output
 Return ONLY the rewritten guidance."""
         headers = {"Content-Type": "application/json"}
-        payload = {"contents": [{"parts": [{"text": enhanced_prompt}]}]}
+        payload = {"contents": [{"parts": [{"text": enhanced_prompt}]}]]}
         bot_text = ""
         http_code = 0
         if use_gemini and GEMINI_API_KEY:
@@ -1548,7 +1471,7 @@ async def generate_ticket_desc_ep(input_data: TicketDescInput):
             "Do not include any extra text, comments, or explanations outside the JSON."
         )
         headers = {"Content-Type": "application/json"}
-        payload = {"contents": [{"parts": [{"text": prompt}]}]}
+        payload = {"contents": [{"parts": [{"text": prompt}]}]]}
         resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=25, verify=GEMINI_SSL_VERIFY)
         try:
             data = resp.json()