rohitsar567 Claude Opus 4.7 (1M context) commited on
Commit
a5d9bfd
Β·
1 Parent(s): f266901

fix(#40+voice+upload): SSOT scorecard parity + Sarvam 30s-truncation + uploaded-PDF retrieval gate

Browse files

#40 SINGLE SOURCE OF TRUTH: extract the marketplace catalogue into
backend.main._marketplace_catalogue + marketplace_grade(); brain_tools
._scorecard_signal now delegates (deleted the parallel doctype-rank/_merge
re-implementation). Rec-card grade == marketplace grade for all 148 by
construction β€” deterministic oracle 790 ids / 0 mismatches; parity test
rewritten to assert the full id universe + alias/doctype edges.

VOICE: Sarvam saarika REST has a hard ~30s limit and silently returns
only the first 30s (HTTP 200). sarvam_stt now decodes to 16k mono and
splits >25s audio on silence, transcribes sequentially, concatenates,
raises on error (no silent failure). Frontend POST timeout scales with
blob size. Regression test: 92s utterance -> all 92 words.

UPLOAD: brain_tools profile-gate silently returned 0 chunks for a freshly
uploaded PDF asked about before fact-find (contradicted the 'searchable
now' UI promise). Added session-scoped quarantine bypass + single_brain
UPLOADED-DOC prompt rule. Session-private, no cross-session leak.

Full pytest gate green (rc=0); frontend compiled clean.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/brain_tools.py CHANGED
@@ -261,99 +261,33 @@ def _insurer_reviews(slug: str) -> Optional[dict]:
261
  return ir
262
 
263
 
264
- # Mirrors the LOCAL _DOCTYPE_RANK inside backend.main.policies_all β€” the
265
- # marketplace picks, per product, the extracted file with the best doctype
266
- # rank. _scorecard_signal MUST select the identical file or the cited-card
267
- # grade diverges from the marketplace card for multi-doctype products
268
- # (#40). Keep in sync with main.policies_all if that ever changes.
269
- _DOCTYPE_RANK = {"wordings": 0, "prospectus": 1, "cis": 2, "brochure": 3}
270
-
271
-
272
  def _scorecard_signal(policy_id: str, profile=None) -> dict:
273
- """{_grade, _overall_score} from backend.scorecard using the same
274
- inputs as the marketplace, so a policy's recommendation grade equals
275
- its marketplace grade (asserted by tests/test_scorecard_parity.py).
276
- Read-only; returns {} on any failure (ranking still works off co-pay
277
- + SI headroom + cosine)."""
 
 
 
 
 
 
 
 
 
 
 
 
278
  try:
279
- cur = _curated_facts_all()
280
- # PARITY-BY-CONSTRUCTION (#40, 2026-05-18): resolve the EXACT data
281
- # the marketplace card grades. main.policies_all picks, per product
282
- # key, the extracted file with the best doctype rank (wordings >
283
- # prospectus > cis > brochure), then _merge_curated(extracted,
284
- # curated[extracted.policy_id or pid] or curated[pid]). Taking the
285
- # FIRST _candidate_stems hit instead graded the PASSED id's own
286
- # doctype, so a __cis/__brochure recommendation id scored a
287
- # different file than the marketplace's canonical __wordings card
288
- # (royal-sundaram lifeline A↔C, sbi super-top-up B↔A,
289
- # new-india-mediclaim C↔D). Mirror the marketplace selection so
290
- # rec grade == marketplace grade for ALL 148 by construction.
291
- pid = (policy_id or "").strip()
292
- pkey = pid.rsplit("__", 1)[0] if "__" in pid else pid
293
- best = None # ((rank, stem), Path)
294
- try:
295
- for fp in settings.EXTRACTED_DIR.glob("*.json"):
296
- st = fp.stem
297
- fk = st.rsplit("__", 1)[0] if "__" in st else st
298
- if fk != pkey:
299
- continue
300
- dt = st.rsplit("__", 1)[1] if "__" in st else ""
301
- rk = (_DOCTYPE_RANK.get(dt, 99), st)
302
- if best is None or rk < best[0]:
303
- best = (rk, fp)
304
- except Exception: # noqa: BLE001 β€” glob must never break ranking
305
- best = None
306
- extracted = None
307
- if best is not None:
308
- try:
309
- extracted = _json.loads(best[1].read_text())
310
- except Exception: # noqa: BLE001 β€” corrupt extract β†’ curated-only
311
- extracted = None
312
- # Curated entry resolved EXACTLY as the marketplace merge does
313
- # (curated[extracted.policy_id or pid] or curated[pid]); fall back
314
- # to the candidate-stem scan only when there is no extracted file.
315
- curated_entry = None
316
- if extracted is not None:
317
- curated_entry = cur.get(extracted.get("policy_id") or pid) or cur.get(pid)
318
- if curated_entry is None:
319
- for stem in _candidate_stems(pid):
320
- if stem in cur:
321
- curated_entry = cur[stem]
322
- break
323
- if extracted is not None:
324
- from backend.main import _merge_curated # lazy: avoids cycle
325
- data = _merge_curated(extracted, curated_entry)
326
- elif curated_entry:
327
- data = curated_entry
328
- else:
329
- # Genuinely unknown policy β†’ fail OPEN (no false weak signal;
330
- # _recommendation_fit keeps chunks with no grade evidence).
331
- return {}
332
- from backend.scorecard import build_scorecard
333
-
334
- slug = data.get("insurer_slug") or (
335
- policy_id.split("__", 1)[0] if "__" in policy_id else ""
336
- )
337
- ir = _insurer_reviews(slug)
338
- prof = None
339
- if profile is not None:
340
- prof = {
341
- s: getattr(profile, s, None)
342
- for s in (
343
- "age", "income_band", "primary_goal",
344
- "existing_cover_inr", "copay_pct",
345
- )
346
  }
347
- sc = build_scorecard(data, insurer_reviews=ir, profile=prof)
348
- # Scorecard.overall_score is the numeric field; `overall` is a
349
- # back-compat alias.
350
- _overall = getattr(sc, "overall_score", None)
351
- if _overall is None:
352
- _overall = getattr(sc, "overall", None)
353
- return {
354
- "_grade": getattr(sc, "grade", None),
355
- "_overall_score": _overall,
356
- }
357
  except Exception: # noqa: BLE001 β€” scorecard optional; ranking degrades gracefully
358
  return {}
359
 
@@ -765,6 +699,100 @@ def _quality_seed_candidates(profile, limit: int = 25) -> list[dict]:
765
  return out[:limit]
766
 
767
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
768
  # ---- retrieve_policies -----------------------------------------------------
769
 
770
  async def retrieve_policies(
@@ -813,6 +841,38 @@ async def retrieve_policies(
813
  if not isinstance(query, str) or not query.strip():
814
  return {"chunks": [], "count": 0, "error": "empty_query"}
815
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
816
  # Profile-complete gate (skip if caller supplied policy_filter_ids β€” that
817
  # branch is a known-policy follow-up). Defense-in-depth: even if the LLM
818
  # ignores RULE 2 of the system prompt, this refuses to return chunks
 
261
  return ir
262
 
263
 
 
 
 
 
 
 
 
 
264
  def _scorecard_signal(policy_id: str, profile=None) -> dict:
265
+ """{_grade, _overall_score} for policy_id β€” the SAME grade its
266
+ marketplace card shows (#40 SINGLE SOURCE OF TRUTH, 2026-05-18).
267
+
268
+ Delegates to backend.main.marketplace_grade, which derives the grade
269
+ from the ONE marketplace catalogue computation
270
+ (backend.main._marketplace_catalogue) using UIN-canonical resolution:
271
+ a marketing-rename / KI-145 variant id resolves onto its canonical
272
+ card. There is no longer a parallel doctype-rank/_merge
273
+ re-implementation here, so the cited-card grade cannot drift from the
274
+ marketplace card grade β€” for ALL 148 by construction, asserted by
275
+ tests/test_scorecard_parity.py.
276
+
277
+ `profile` is accepted for call-site compatibility. The parity-relevant
278
+ grade LETTER (what _recommendation_fit gates on) is the profile-neutral
279
+ marketplace grade, identical to the /api/policies/all catalogue view.
280
+ Read-only; returns {} only when the policy is unknown to the
281
+ marketplace (ranking still works off co-pay + SI headroom + cosine)."""
282
  try:
283
+ from backend.main import marketplace_grade # lazy: avoids cycle
284
+ sig = marketplace_grade(policy_id)
285
+ if sig and sig.get("_grade"):
286
+ return {
287
+ "_grade": sig.get("_grade"),
288
+ "_overall_score": sig.get("_overall_score"),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  }
290
+ return {}
 
 
 
 
 
 
 
 
 
291
  except Exception: # noqa: BLE001 β€” scorecard optional; ranking degrades gracefully
292
  return {}
293
 
 
699
  return out[:limit]
700
 
701
 
702
+ # ---- uploaded-doc (quarantine) bypass helpers ------------------------------
703
+
704
+
705
+ def _session_has_quarantine_docs(session_id: str) -> bool:
706
+ """True iff this session has at least one chunk in the SEPARATE
707
+ `user_uploads_quarantine` Chroma collection.
708
+
709
+ Strictly session-scoped (where={"session_id": session_id}); cheap
710
+ metadata-only `.get(limit=1)`. Never raises β€” a Chroma hiccup just
711
+ means "treat as no upload" so the normal profile-gate path runs.
712
+ """
713
+ if not session_id:
714
+ return False
715
+ try:
716
+ from rag.ingest import get_quarantine_collection
717
+
718
+ q = get_quarantine_collection().get(
719
+ where={"session_id": session_id},
720
+ limit=1,
721
+ include=[],
722
+ )
723
+ return bool(q and q.get("ids"))
724
+ except Exception as e: # noqa: BLE001 β€” best-effort probe
725
+ _log.warning(
726
+ "quarantine presence probe failed (sid=%s): %s: %s",
727
+ session_id, type(e).__name__, str(e)[:160],
728
+ )
729
+ return False
730
+
731
+
732
+ async def _retrieve_uploaded_only(
733
+ query: str, session_id: str, top_k: int
734
+ ) -> dict:
735
+ """Quarantine-scoped retrieval that returns ONLY this session's
736
+ uploaded-doc chunks, bypassing the recommendation profile-gate.
737
+
738
+ Reuses `rag.retrieve.retrieve(session_id=...)` (whose quarantine boost
739
+ pass already prepends this session's uploaded chunks), then filters the
740
+ result down to `doc_type == "user_upload"` so the bypass response can
741
+ NEVER contain general-corpus policy chunks against an incomplete
742
+ profile. Returns the same dict shape as `retrieve_policies`.
743
+ """
744
+ try:
745
+ from rag.retrieve import retrieve as _retrieve
746
+
747
+ chunks = await _retrieve(
748
+ query=query,
749
+ top_k=max(int(top_k) if top_k else 8, 3),
750
+ session_id=session_id,
751
+ )
752
+ except Exception as e: # noqa: BLE001 β€” graceful empty
753
+ _log.warning(
754
+ "uploaded-only retrieve failed (sid=%s): %s: %s",
755
+ session_id, type(e).__name__, str(e)[:160],
756
+ )
757
+ return {"chunks": [], "count": 0, "error": f"{type(e).__name__}"}
758
+
759
+ uploaded = []
760
+ for c in chunks or []:
761
+ if (getattr(c, "doc_type", "") or "").lower() != "user_upload":
762
+ continue
763
+ # Triple-check session ownership before exposing the chunk β€”
764
+ # belt + suspenders on top of the where={"session_id"} filter.
765
+ if (getattr(c, "session_id", None) or session_id) != session_id and \
766
+ getattr(c, "session_id", None) not in (None, ""):
767
+ continue
768
+ uploaded.append({
769
+ "chunk_id": getattr(c, "chunk_id", ""),
770
+ "policy_id": getattr(c, "policy_id", ""),
771
+ "policy_name": getattr(c, "policy_name", ""),
772
+ "insurer_slug": getattr(c, "insurer_slug", "") or "user-upload",
773
+ "doc_type": "user_upload",
774
+ "source_url": getattr(c, "source_url", ""),
775
+ "chunk_text": (getattr(c, "text", "") or "")[:1200],
776
+ "score": float(getattr(c, "score", 0.0) or 0.0),
777
+ })
778
+ if not uploaded:
779
+ return {"chunks": [], "count": 0}
780
+ return {
781
+ "chunks": uploaded,
782
+ "count": len(uploaded),
783
+ "query": query,
784
+ "source": "uploaded_doc_quarantine",
785
+ "note": (
786
+ "These chunks come from the user's OWN uploaded policy PDF "
787
+ "(session-scoped quarantine). Answer the user's question "
788
+ "about THIS document directly. Do NOT treat this as a "
789
+ "general recommendation and do NOT block on profile "
790
+ "completeness β€” the user explicitly asked about their "
791
+ "uploaded file."
792
+ ),
793
+ }
794
+
795
+
796
  # ---- retrieve_policies -----------------------------------------------------
797
 
798
  async def retrieve_policies(
 
841
  if not isinstance(query, str) or not query.strip():
842
  return {"chunks": [], "count": 0, "error": "empty_query"}
843
 
844
+ # UPLOADED-DOC BYPASS (2026-05-18) β€” the user is explicitly promised in
845
+ # the UI ("βœ“ Indexed … It's now searchable in this chat. Ask me about
846
+ # it.") that an uploaded policy PDF is immediately queryable. But the
847
+ # profile-complete gate below blocks ALL retrieval until the 7-slot
848
+ # fact-find is done (unless policy_filter_ids is set, which the LLM
849
+ # can't know for a freshly-uploaded doc). An uploaded doc is, by
850
+ # definition, a known-doc-FOR-THIS-SESSION β€” semantically identical to
851
+ # the policy_filter_ids follow-up branch that already legitimately
852
+ # bypasses the gate. So: when this session has chunks in the SEPARATE
853
+ # `user_uploads_quarantine` collection, run a quarantine-ONLY retrieval
854
+ # that bypasses the recommendation profile-gate. Strictly session-
855
+ # scoped (where={"session_id": eff_session_id}) so user A's upload
856
+ # never leaks into user B. The full recommendation flow (general
857
+ # corpus + eligibility + scorecard) still requires the complete
858
+ # profile β€” this bypass only surfaces the user's OWN uploaded doc.
859
+ if (
860
+ eff_session_id
861
+ and not policy_filter_ids
862
+ and _session_has_quarantine_docs(eff_session_id)
863
+ ):
864
+ up = await _retrieve_uploaded_only(query, eff_session_id, top_k)
865
+ # Only short-circuit if the gate would otherwise block (incomplete
866
+ # profile) AND we actually found uploaded chunks. If the profile is
867
+ # already complete we fall through so the upload is folded into the
868
+ # normal ranked recommendation pool by the quarantine boost pass.
869
+ _profile_incomplete = profile is not None and any(
870
+ getattr(profile, slot, None) in (None, "", [])
871
+ for slot in _REQUIRED_FOR_READY
872
+ )
873
+ if up.get("count", 0) > 0 and _profile_incomplete:
874
+ return up
875
+
876
  # Profile-complete gate (skip if caller supplied policy_filter_ids β€” that
877
  # branch is a known-policy follow-up). Defense-in-depth: even if the LLM
878
  # ignores RULE 2 of the system prompt, this refuses to return chunks
backend/main.py CHANGED
@@ -2936,33 +2936,18 @@ def _ki145_material_diffs(curated: dict, extracted: dict) -> int:
2936
  return diffs
2937
 
2938
 
2939
- @app.get("/api/policies/all", response_model=MarketplaceResponse)
2940
- async def policies_all(session_id: Optional[str] = None):
2941
- """The marketplace data feed β€” every extracted policy + scorecard + filterable fields.
2942
-
2943
- When session_id is provided AND the session has a profile populated to
2944
- β‰₯0.6 completeness, every policy is scored against THAT profile (dynamic
2945
- per-user grade). Otherwise we score with the generic baseline weights.
 
 
2946
  """
2947
  import json as _json
2948
- from backend.scorecard import build_scorecard, profile_completeness as _completeness
2949
- from backend.session_state import get_session as _get_sess
2950
-
2951
- # Pull user profile if we have one
2952
- user_profile_dict: Optional[dict] = None
2953
- if session_id:
2954
- sess = _get_sess(session_id)
2955
- p = sess.profile
2956
- profile_dict = {
2957
- "age": p.age, "dependents": p.dependents, "income_band": p.income_band,
2958
- "existing_cover_inr": p.existing_cover_inr, "primary_goal": p.primary_goal,
2959
- "location_tier": p.location_tier, "parents_to_insure": p.parents_to_insure,
2960
- "parents_age_max": p.parents_age_max, "parents_has_ped": p.parents_has_ped,
2961
- "health_conditions": p.health_conditions, "budget_band": p.budget_band,
2962
- }
2963
- if _completeness(profile_dict) >= 0.6:
2964
- user_profile_dict = profile_dict
2965
-
2966
  corpus_url_index = _build_corpus_url_index()
2967
  curated_facts = _load_curated_facts()
2968
 
@@ -3485,7 +3470,36 @@ async def policies_all(session_id: Optional[str] = None):
3485
  else:
3486
  prev.aliases = sorted(set(prev.aliases) | set(p.aliases))
3487
  deduped = list(_best.values())
 
 
 
 
 
 
 
 
 
 
 
 
 
3488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3489
  return MarketplaceResponse(
3490
  policies=deduped,
3491
  total=len(deduped),
@@ -4294,3 +4308,117 @@ else:
4294
  "docs": "/docs",
4295
  "health": "/api/health",
4296
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2936
  return diffs
2937
 
2938
 
2939
+ def _marketplace_catalogue(user_profile_dict):
2940
+ """SINGLE SOURCE OF TRUTH for the marketplace card set (#40).
2941
+
2942
+ The recommendation path (brain_tools._scorecard_signal via
2943
+ marketplace_grade) and /api/policies/all BOTH derive a policy's
2944
+ grade from THIS one computation, so the cited-card grade can never
2945
+ diverge from the marketplace card grade β€” including marketing-rename
2946
+ alias / KI-145 variant cards. Body is the former inline endpoint
2947
+ logic, moved verbatim; fully synchronous.
2948
  """
2949
  import json as _json
2950
+ from backend.scorecard import build_scorecard
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2951
  corpus_url_index = _build_corpus_url_index()
2952
  curated_facts = _load_curated_facts()
2953
 
 
3470
  else:
3471
  prev.aliases = sorted(set(prev.aliases) | set(p.aliases))
3472
  deduped = list(_best.values())
3473
+ return deduped
3474
+
3475
+ @app.get("/api/policies/all", response_model=MarketplaceResponse)
3476
+ async def policies_all(session_id: Optional[str] = None):
3477
+ """The marketplace data feed β€” every extracted policy + scorecard + filterable fields.
3478
+
3479
+ When session_id is provided AND the session has a profile populated to
3480
+ β‰₯0.6 completeness, every policy is scored against THAT profile (dynamic
3481
+ per-user grade). Otherwise we score with the generic baseline weights.
3482
+ """
3483
+ import json as _json
3484
+ from backend.scorecard import build_scorecard, profile_completeness as _completeness
3485
+ from backend.session_state import get_session as _get_sess
3486
 
3487
+ # Pull user profile if we have one
3488
+ user_profile_dict: Optional[dict] = None
3489
+ if session_id:
3490
+ sess = _get_sess(session_id)
3491
+ p = sess.profile
3492
+ profile_dict = {
3493
+ "age": p.age, "dependents": p.dependents, "income_band": p.income_band,
3494
+ "existing_cover_inr": p.existing_cover_inr, "primary_goal": p.primary_goal,
3495
+ "location_tier": p.location_tier, "parents_to_insure": p.parents_to_insure,
3496
+ "parents_age_max": p.parents_age_max, "parents_has_ped": p.parents_has_ped,
3497
+ "health_conditions": p.health_conditions, "budget_band": p.budget_band,
3498
+ }
3499
+ if _completeness(profile_dict) >= 0.6:
3500
+ user_profile_dict = profile_dict
3501
+
3502
+ deduped = _marketplace_catalogue(user_profile_dict)
3503
  return MarketplaceResponse(
3504
  policies=deduped,
3505
  total=len(deduped),
 
4308
  "docs": "/docs",
4309
  "health": "/api/health",
4310
  }
4311
+
4312
+
4313
+ # ---- #40 SSOT grade resolver ------------------------------------------------
4314
+ # marketplace_grade(policy_id) returns the SAME (grade, overall) the
4315
+ # marketplace card for that policy's canonical identity shows. The
4316
+ # recommendation path calls this instead of re-deriving a scorecard, so
4317
+ # rec-card grade == marketplace grade for ALL 148 by construction.
4318
+
4319
+ import threading as _mg_threading
4320
+
4321
+ _MG_LOCK = _mg_threading.Lock()
4322
+ _MG_CACHE: dict = {"sig": None, "index": None}
4323
+
4324
+
4325
+ def _mg_data_signature() -> tuple:
4326
+ """Cheap fingerprint that changes when any grading input changes
4327
+ (so an uploaded-PDF card or a curated edit invalidates the cache)."""
4328
+ sig = []
4329
+ for d in (settings.EXTRACTED_DIR, settings.DATA_DIR / "policy_facts",
4330
+ settings.DATA_DIR / "reviews"):
4331
+ try:
4332
+ for fp in sorted(d.glob("*.json")):
4333
+ st = fp.stat()
4334
+ sig.append((fp.name, int(st.st_mtime), st.st_size))
4335
+ except Exception: # noqa: BLE001 β€” missing dir β†’ empty contribution
4336
+ continue
4337
+ return tuple(sig)
4338
+
4339
+
4340
+ def _mg_norm_uin(raw) -> str:
4341
+ try:
4342
+ from backend.policy_identity import normalize_uin
4343
+ return normalize_uin(raw)
4344
+ except Exception: # noqa: BLE001
4345
+ return ""
4346
+
4347
+
4348
+ def _mg_build_index() -> dict:
4349
+ """{lookup_key -> (grade, overall_score)} for every marketplace card,
4350
+ keyed by policy_id, product_key, and normalised UIN so a variant /
4351
+ alias id resolves to its canonical card's grade."""
4352
+ cards = _marketplace_catalogue(None)
4353
+ cur = _load_curated_facts()
4354
+ idx: dict = {}
4355
+
4356
+ def _put(k, val):
4357
+ if k:
4358
+ idx.setdefault(k, val)
4359
+
4360
+ for c in cards:
4361
+ val = (c.grade, c.overall_score)
4362
+ pid = c.policy_id or ""
4363
+ _put(f"id:{pid}", val)
4364
+ try:
4365
+ from backend.policy_identity import product_key as _pk
4366
+ _put(f"pk:{_pk(pid)}", val)
4367
+ except Exception: # noqa: BLE001
4368
+ pass
4369
+ # UIN of the card's underlying data (curated wins, like the catalogue)
4370
+ cdata = cur.get(pid) or {}
4371
+ uin = _mg_norm_uin(cdata.get("uin_code") or cdata.get("uin"))
4372
+ if not uin:
4373
+ try:
4374
+ ep = settings.EXTRACTED_DIR / f"{pid}.json"
4375
+ if ep.exists():
4376
+ import json as _j
4377
+ uin = _mg_norm_uin(_j.loads(ep.read_text()).get("uin_code"))
4378
+ except Exception: # noqa: BLE001
4379
+ uin = ""
4380
+ _put(f"uin:{uin}" if uin else "", val)
4381
+ return idx
4382
+
4383
+
4384
+ def _mg_index() -> dict:
4385
+ sig = _mg_data_signature()
4386
+ with _MG_LOCK:
4387
+ if _MG_CACHE["sig"] != sig or _MG_CACHE["index"] is None:
4388
+ _MG_CACHE["index"] = _mg_build_index()
4389
+ _MG_CACHE["sig"] = sig
4390
+ return _MG_CACHE["index"]
4391
+
4392
+
4393
+ def marketplace_grade(policy_id: str) -> dict:
4394
+ """{"_grade", "_overall_score"} for policy_id, identical to its
4395
+ marketplace card. Resolution order: exact id -> product_key -> UIN
4396
+ (so a marketing-rename / variant id maps onto its canonical card).
4397
+ Returns {} only when the policy is unknown to the marketplace."""
4398
+ if not policy_id:
4399
+ return {}
4400
+ idx = _mg_index()
4401
+ from backend.policy_identity import product_key as _pk
4402
+ cur = _load_curated_facts()
4403
+ pid = policy_id.strip()
4404
+ keys = [f"id:{pid}", f"pk:{_pk(pid)}"]
4405
+ cdata = cur.get(pid) or cur.get(_pk(pid)) or {}
4406
+ uin = _mg_norm_uin(cdata.get("uin_code") or cdata.get("uin"))
4407
+ if not uin:
4408
+ try:
4409
+ import json as _j
4410
+ for cand in (pid, _pk(pid)):
4411
+ ep = settings.EXTRACTED_DIR / f"{cand}.json"
4412
+ if ep.exists():
4413
+ uin = _mg_norm_uin(_j.loads(ep.read_text()).get("uin_code"))
4414
+ if uin:
4415
+ break
4416
+ except Exception: # noqa: BLE001
4417
+ uin = ""
4418
+ if uin:
4419
+ keys.append(f"uin:{uin}")
4420
+ for k in keys:
4421
+ if k in idx:
4422
+ g, o = idx[k]
4423
+ return {"_grade": g, "_overall_score": o}
4424
+ return {}
backend/providers/sarvam_stt.py CHANGED
@@ -9,13 +9,34 @@ Response: {"transcript": str, "language_code": str?, "language_probability": flo
9
  from __future__ import annotations
10
 
11
  import io
12
- from typing import Optional
 
13
 
14
  import httpx
15
 
16
  from backend.config import settings
17
  from backend.providers.base import STTProvider, STTResult
18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
 
20
  # Public error-code vocabulary returned by /api/transcribe when Sarvam fails.
21
  # Frontends (PTT + live voice) consume these as a closed enum so they can map
@@ -103,8 +124,8 @@ class SarvamSTT(STTProvider):
103
  _SARVAM_NATIVE_FORMATS = {"wav", "mp3", "flac", "ogg", "m4a"}
104
 
105
  @staticmethod
106
- def _transcode_to_wav(audio_bytes: bytes, src_format: str) -> bytes:
107
- """Convert any pydub-readable container to 16 kHz mono WAV.
108
 
109
  Sarvam's recommended sampling rate is 16 kHz mono β€” what Saarika
110
  was trained on. Down-mixing + resampling at the gateway also
@@ -112,11 +133,101 @@ class SarvamSTT(STTProvider):
112
  """
113
  from pydub import AudioSegment
114
  audio = AudioSegment.from_file(io.BytesIO(audio_bytes), format=src_format)
115
- audio = audio.set_frame_rate(16000).set_channels(1).set_sample_width(2)
 
 
 
116
  buf = io.BytesIO()
117
  audio.export(buf, format="wav")
118
  return buf.getvalue()
119
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
  async def transcribe(
121
  self,
122
  audio_bytes: bytes,
@@ -129,36 +240,105 @@ class SarvamSTT(STTProvider):
129
  return STTResult(text="", language_code=language_code, confidence=0.0, raw={"reason": "audio_too_short"})
130
 
131
  fmt = (audio_format or "wav").lower().lstrip(".")
132
- # Browser MediaRecorder uses webm/opus by default; Sarvam rejects it.
133
- # Transcode in-process when we get a non-native format.
134
- if fmt not in self._SARVAM_NATIVE_FORMATS:
135
- try:
136
- audio_bytes = self._transcode_to_wav(audio_bytes, fmt)
137
- fmt = "wav"
138
- except Exception as e:
139
- # If pydub/ffmpeg fails (e.g., truly corrupt audio), let Sarvam
140
- # see the original bytes and return its own error rather than
141
- # silently swallowing.
142
- pass
143
 
144
- url = f"{settings.SARVAM_BASE_URL}{settings.SARVAM_STT_PATH}"
145
- files = {
146
- "file": (f"audio.{fmt}", io.BytesIO(audio_bytes), f"audio/{fmt}"),
147
- }
148
- data = {"model": self.model}
149
- if language_code:
150
- data["language_code"] = language_code
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
 
152
- headers = {"api-subscription-key": self.api_key}
 
 
 
 
 
153
 
154
- async with httpx.AsyncClient(timeout=self.timeout) as client:
155
- resp = await client.post(url, headers=headers, files=files, data=data)
156
- resp.raise_for_status()
157
- payload = resp.json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
158
 
 
159
  return STTResult(
160
- text=payload.get("transcript", ""),
161
- language_code=payload.get("language_code"),
162
- confidence=payload.get("language_probability"),
163
- raw=payload,
 
 
 
 
 
164
  )
 
9
  from __future__ import annotations
10
 
11
  import io
12
+ import logging
13
+ from typing import List, Optional
14
 
15
  import httpx
16
 
17
  from backend.config import settings
18
  from backend.providers.base import STTProvider, STTResult
19
 
20
+ _log = logging.getLogger(__name__)
21
+
22
+ # Sarvam's saarika REST /speech-to-text endpoint has a hard ~30s audio
23
+ # limit. Critically, it does NOT 4xx on longer audio β€” it returns HTTP 200
24
+ # with a `transcript` containing ONLY the first ~30s and silently drops the
25
+ # rest. The live-voice hook's grace-window batching (UTTERANCE_GRACE_MS)
26
+ # deliberately merges multiple pause-separated speech bursts into ONE blob,
27
+ # so any real-world utterance with natural pauses easily exceeds 30s of
28
+ # wall-clock audio and gets truncated. To capture the COMPLETE utterance we
29
+ # split anything over the safe ceiling into <= SAFE chunks (at silence
30
+ # boundaries where possible so words aren't cut mid-syllable), transcribe
31
+ # each chunk, and concatenate the transcripts in order.
32
+ #
33
+ # Ceiling is set below the documented 30s so the boundary search + minor
34
+ # encode jitter never pushes a chunk back over Sarvam's real limit.
35
+ STT_CHUNK_MS = 25_000 # target max length per Sarvam call
36
+ STT_SILENCE_SEARCH_MS = 4_000 # widen a cut by up to this to land on silence
37
+ STT_MIN_SILENCE_MS = 350 # a gap >= this counts as a sentence pause
38
+ STT_MAX_TOTAL_MS = 15 * 60_000 # hard safety cap (15 min) β€” refuse absurd input
39
+
40
 
41
  # Public error-code vocabulary returned by /api/transcribe when Sarvam fails.
42
  # Frontends (PTT + live voice) consume these as a closed enum so they can map
 
124
  _SARVAM_NATIVE_FORMATS = {"wav", "mp3", "flac", "ogg", "m4a"}
125
 
126
  @staticmethod
127
+ def _load_segment(audio_bytes: bytes, src_format: str):
128
+ """Decode any pydub-readable container to a 16 kHz mono AudioSegment.
129
 
130
  Sarvam's recommended sampling rate is 16 kHz mono β€” what Saarika
131
  was trained on. Down-mixing + resampling at the gateway also
 
133
  """
134
  from pydub import AudioSegment
135
  audio = AudioSegment.from_file(io.BytesIO(audio_bytes), format=src_format)
136
+ return audio.set_frame_rate(16000).set_channels(1).set_sample_width(2)
137
+
138
+ @staticmethod
139
+ def _segment_to_wav_bytes(audio) -> bytes:
140
  buf = io.BytesIO()
141
  audio.export(buf, format="wav")
142
  return buf.getvalue()
143
 
144
+ @classmethod
145
+ def _transcode_to_wav(cls, audio_bytes: bytes, src_format: str) -> bytes:
146
+ """Convert any pydub-readable container to 16 kHz mono WAV."""
147
+ return cls._segment_to_wav_bytes(cls._load_segment(audio_bytes, src_format))
148
+
149
+ @staticmethod
150
+ def _split_on_silence(audio, chunk_ms: int) -> List:
151
+ """Split `audio` into <= chunk_ms slices.
152
+
153
+ We never cut a slice longer than chunk_ms. Where possible the cut is
154
+ nudged BACKWARDS to the quietest point within the last
155
+ STT_SILENCE_SEARCH_MS of the slice so the boundary lands in a pause
156
+ rather than mid-word (which would drop a syllable from BOTH the tail
157
+ of one chunk and the head of the next). If no clear pause is found we
158
+ cut at the hard chunk_ms boundary β€” a worst-case half-word seam, far
159
+ better than silently losing every word past 30s.
160
+ """
161
+ total = len(audio)
162
+ if total <= chunk_ms:
163
+ return [audio]
164
+
165
+ # Cheap energy probe per 100ms frame so we can find the quietest
166
+ # spot near each prospective cut without an external VAD.
167
+ from pydub import AudioSegment # noqa: F401 (ensures pydub import path)
168
+
169
+ chunks: List = []
170
+ start = 0
171
+ while start < total:
172
+ if total - start <= chunk_ms:
173
+ chunks.append(audio[start:])
174
+ break
175
+
176
+ hard_cut = start + chunk_ms
177
+ search_lo = max(start + 1, hard_cut - STT_SILENCE_SEARCH_MS)
178
+ window = audio[search_lo:hard_cut]
179
+
180
+ # Find the 100ms frame with the lowest dBFS in the search window.
181
+ best_offset = None
182
+ best_db = None
183
+ step = 100
184
+ for off in range(0, max(1, len(window)), step):
185
+ frame = window[off:off + step]
186
+ if len(frame) == 0:
187
+ continue
188
+ db = frame.dBFS # -inf for pure digital silence
189
+ if best_db is None or db < best_db:
190
+ best_db = db
191
+ best_offset = off
192
+
193
+ # Only honour the silence cut if the quietest frame is actually
194
+ # quiet relative to the segment (a real pause), else hard-cut.
195
+ cut = hard_cut
196
+ if (
197
+ best_offset is not None
198
+ and best_db is not None
199
+ and (best_db == float("-inf") or best_db < audio.dBFS - 12)
200
+ ):
201
+ cut = search_lo + best_offset + step // 2
202
+ cut = min(max(cut, start + 1), hard_cut)
203
+
204
+ chunks.append(audio[start:cut])
205
+ start = cut
206
+ return chunks
207
+
208
+ async def _transcribe_wav_bytes(
209
+ self,
210
+ wav_bytes: bytes,
211
+ language_code: Optional[str],
212
+ ) -> dict:
213
+ """POST one already-encoded WAV blob to Sarvam, return its JSON.
214
+
215
+ Raises on transport / HTTP errors so the caller's classifier can map
216
+ them to a closed error_code β€” we deliberately do NOT swallow here.
217
+ """
218
+ url = f"{settings.SARVAM_BASE_URL}{settings.SARVAM_STT_PATH}"
219
+ files = {
220
+ "file": ("audio.wav", io.BytesIO(wav_bytes), "audio/wav"),
221
+ }
222
+ data = {"model": self.model}
223
+ if language_code:
224
+ data["language_code"] = language_code
225
+ headers = {"api-subscription-key": self.api_key}
226
+ async with httpx.AsyncClient(timeout=self.timeout) as client:
227
+ resp = await client.post(url, headers=headers, files=files, data=data)
228
+ resp.raise_for_status()
229
+ return resp.json()
230
+
231
  async def transcribe(
232
  self,
233
  audio_bytes: bytes,
 
240
  return STTResult(text="", language_code=language_code, confidence=0.0, raw={"reason": "audio_too_short"})
241
 
242
  fmt = (audio_format or "wav").lower().lstrip(".")
 
 
 
 
 
 
 
 
 
 
 
243
 
244
+ # Decode to a 16 kHz mono AudioSegment so we can (a) measure the true
245
+ # duration and (b) chunk anything over Sarvam's ~30s REST limit. The
246
+ # legacy path only transcoded webm→wav; native containers (wav/mp3/
247
+ # m4a/...) went straight through UNMEASURED, so a 90s m4a was sent
248
+ # whole and silently truncated to its first 30s by Sarvam. Decoding
249
+ # every format here closes that hole for ALL callers (live voice +
250
+ # push-to-talk).
251
+ segment = None
252
+ try:
253
+ segment = self._load_segment(audio_bytes, fmt)
254
+ except Exception:
255
+ # pydub/ffmpeg couldn't decode (truly corrupt audio, or a codec
256
+ # ffmpeg lacks). Fall back to the original single-shot path so
257
+ # Sarvam returns its own error rather than us swallowing it.
258
+ # Native formats are sent as-is; unknown ones default to wav.
259
+ _log.warning(
260
+ "STT decode failed (fmt=%s, %d bytes) β€” single-shot fallback",
261
+ fmt,
262
+ len(audio_bytes),
263
+ )
264
+ send_fmt = fmt if fmt in self._SARVAM_NATIVE_FORMATS else "wav"
265
+ url = f"{settings.SARVAM_BASE_URL}{settings.SARVAM_STT_PATH}"
266
+ files = {
267
+ "file": (f"audio.{send_fmt}", io.BytesIO(audio_bytes), f"audio/{send_fmt}"),
268
+ }
269
+ data = {"model": self.model}
270
+ if language_code:
271
+ data["language_code"] = language_code
272
+ headers = {"api-subscription-key": self.api_key}
273
+ async with httpx.AsyncClient(timeout=self.timeout) as client:
274
+ resp = await client.post(url, headers=headers, files=files, data=data)
275
+ resp.raise_for_status()
276
+ payload = resp.json()
277
+ return STTResult(
278
+ text=payload.get("transcript", ""),
279
+ language_code=payload.get("language_code"),
280
+ confidence=payload.get("language_probability"),
281
+ raw=payload,
282
+ )
283
 
284
+ total_ms = len(segment)
285
+ if total_ms > STT_MAX_TOTAL_MS:
286
+ # Refuse absurd input loudly rather than burn N Sarvam calls.
287
+ raise ValueError(
288
+ f"audio too long for STT: {total_ms} ms > cap {STT_MAX_TOTAL_MS} ms"
289
+ )
290
 
291
+ chunks = self._split_on_silence(segment, STT_CHUNK_MS)
292
+
293
+ # Common case: short utterance, one Sarvam call (identical wire
294
+ # behaviour to the pre-fix path, just always WAV-normalised).
295
+ if len(chunks) == 1:
296
+ payload = await self._transcribe_wav_bytes(
297
+ self._segment_to_wav_bytes(chunks[0]), language_code
298
+ )
299
+ return STTResult(
300
+ text=payload.get("transcript", ""),
301
+ language_code=payload.get("language_code"),
302
+ confidence=payload.get("language_probability"),
303
+ raw=payload,
304
+ )
305
+
306
+ # Long utterance: transcribe every chunk, then stitch the
307
+ # transcripts back together in order so the COMPLETE utterance
308
+ # survives. If any chunk fails we let the exception propagate so the
309
+ # /api/transcribe classifier surfaces a real error_code β€” we never
310
+ # return a silently-partial transcript (no silent failures).
311
+ _log.info(
312
+ "STT chunked transcription: %d ms split into %d chunks (<= %d ms each)",
313
+ total_ms,
314
+ len(chunks),
315
+ STT_CHUNK_MS,
316
+ )
317
+ parts: List[str] = []
318
+ first_lang: Optional[str] = None
319
+ first_conf: Optional[float] = None
320
+ raws: List[dict] = []
321
+ for idx, ch in enumerate(chunks):
322
+ payload = await self._transcribe_wav_bytes(
323
+ self._segment_to_wav_bytes(ch), language_code
324
+ )
325
+ piece = (payload.get("transcript") or "").strip()
326
+ if piece:
327
+ parts.append(piece)
328
+ if idx == 0:
329
+ first_lang = payload.get("language_code")
330
+ first_conf = payload.get("language_probability")
331
+ raws.append(payload)
332
 
333
+ stitched = " ".join(parts).strip()
334
  return STTResult(
335
+ text=stitched,
336
+ language_code=first_lang or language_code,
337
+ confidence=first_conf,
338
+ raw={
339
+ "chunked": True,
340
+ "chunk_count": len(chunks),
341
+ "total_ms": total_ms,
342
+ "chunks": raws,
343
+ },
344
  )
backend/single_brain.py CHANGED
@@ -112,6 +112,24 @@ If retrieve_policies returns nothing for that name, say "I couldn't find
112
  that policy in our index. Let me suggest some alternatives" and call
113
  retrieve_policies with a broader query based on the profile.
114
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
  ═══════════════════════════════════
116
  RULE 1 (HIGHEST PRIORITY) β€” save_profile_field is MANDATORY
117
  ═══════════════════════════════════
 
112
  that policy in our index. Let me suggest some alternatives" and call
113
  retrieve_policies with a broader query based on the profile.
114
 
115
+ ═══════════════════════════════════
116
+ UPLOADED-DOC RULE β€” USER-UPLOADED POLICY PDF (answer now, no fact-find gate)
117
+ ═══════════════════════════════════
118
+ The user can upload their own policy PDF (a πŸ“Ž control in the chat). When
119
+ they do, the UI tells them it is "searchable in this chat". If the user
120
+ asks ANYTHING about their uploaded / attached document β€” e.g. "what does my
121
+ policy cover?", "the PDF I just uploaded", "my current plan's room rent",
122
+ "check the file I attached" β€” call retrieve_policies with their question as
123
+ the query (policy_filter_ids=None). This works even if the profile fact-
124
+ find is NOT complete: a tool result with "source": "uploaded_doc_quarantine"
125
+ contains chunks from THEIR OWN uploaded file. Answer the question about
126
+ that document directly and cite it as [Source: <their file's policy name>
127
+ (uploaded document)]. Do NOT block on profile completeness and do NOT ask
128
+ the 7 fact-find questions just to answer a question about their uploaded
129
+ doc. (You still need the full profile before making NEW market
130
+ recommendations β€” see RULE 2 β€” but reading back their own uploaded policy
131
+ is not a recommendation.)
132
+
133
  ═══════════════════════════════════
134
  RULE 1 (HIGHEST PRIORITY) β€” save_profile_field is MANDATORY
135
  ═══════════════════════════════════
frontend/src/lib/useStreamingVoice.ts CHANGED
@@ -914,16 +914,44 @@ export function useStreamingVoice(
914
  // exponential-backoff retry (1s/2s/4s, max 3 attempts). The
915
  // accumulatedText (Web Speech fallback) and accumulated chunks
916
  // are already captured locally, so retries don't lose the
917
- // partial transcript. Each attempt enforces its own 8s timeout
918
  // via the controller signal passed in by retryPostTranscribe.
919
- console.debug("[useStreamingVoice] POST /api/transcribe", { bytes: blob.size, mime: blob.type, lang: language });
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
920
  const sarvam = await retryPostTranscribe(async (signal) => {
921
- // Race per-attempt 8s timeout against the retry signal so a
922
  // hung connection still surfaces as an attempt failure (and
923
  // triggers the next backoff step) rather than blocking
924
  // forever. signal aborts when the OUTER retry loop is killed.
925
  const timeoutCtl = new AbortController();
926
- const timer = setTimeout(() => timeoutCtl.abort(), 8000);
927
  const onOuterAbort = () => timeoutCtl.abort();
928
  signal.addEventListener("abort", onOuterAbort);
929
  try {
 
914
  // exponential-backoff retry (1s/2s/4s, max 3 attempts). The
915
  // accumulatedText (Web Speech fallback) and accumulated chunks
916
  // are already captured locally, so retries don't lose the
917
+ // partial transcript. Each attempt enforces its own timeout
918
  // via the controller signal passed in by retryPostTranscribe.
919
+ //
920
+ // KI-302 (2026-05-18) β€” full-transcript fix. The backend now
921
+ // SPLITS audio over Sarvam's ~30s REST limit into multiple
922
+ // chunks and transcribes them sequentially (one Sarvam round
923
+ // trip per ~25s of speech) so a long utterance is no longer
924
+ // silently truncated to its first 30s. A fixed 8s client
925
+ // timeout would abort that legitimately-longer multi-chunk
926
+ // call mid-flight and force a fall back to the (also often
927
+ // truncated) Web Speech text β€” re-introducing the very bug we
928
+ // are fixing. Scale the per-attempt timeout with the audio
929
+ // size: an 8s floor for short clips plus a generous budget per
930
+ // estimated 25s chunk (browser webm/opus β‰ˆ 4 KB/s β‡’ ~100 KB
931
+ // per 25s chunk; allow ~10s of Sarvam latency per chunk).
932
+ const APPROX_BYTES_PER_CHUNK = 100_000; // ~25s of webm/opus
933
+ const estChunks = Math.max(
934
+ 1,
935
+ Math.ceil(blob.size / APPROX_BYTES_PER_CHUNK),
936
+ );
937
+ const attemptTimeoutMs = Math.min(
938
+ 120_000, // hard ceiling β€” never wait > 2 min on one attempt
939
+ 8_000 + estChunks * 12_000,
940
+ );
941
+ console.debug("[useStreamingVoice] POST /api/transcribe", {
942
+ bytes: blob.size,
943
+ mime: blob.type,
944
+ lang: language,
945
+ estChunks,
946
+ attemptTimeoutMs,
947
+ });
948
  const sarvam = await retryPostTranscribe(async (signal) => {
949
+ // Race per-attempt timeout against the retry signal so a
950
  // hung connection still surfaces as an attempt failure (and
951
  // triggers the next backoff step) rather than blocking
952
  // forever. signal aborts when the OUTER retry loop is killed.
953
  const timeoutCtl = new AbortController();
954
+ const timer = setTimeout(() => timeoutCtl.abort(), attemptTimeoutMs);
955
  const onOuterAbort = () => timeoutCtl.abort();
956
  signal.addEventListener("abort", onOuterAbort);
957
  try {
tests/test_scorecard_parity.py CHANGED
@@ -1,22 +1,32 @@
1
  """Regression guard β€” recommendation-path scorecard == marketplace scorecard.
2
 
3
- ROOT CAUSE THIS LOCKS DOWN (KI-FULLFACTS, 2026-05-17)
4
- -----------------------------------------------------
5
- `brain_tools._scorecard_signal` used to feed `build_scorecard` the 7-key
6
- eligibility subset (`_load_policy_facts`), which is below build_scorecard's
7
- data-completeness floor β†’ every recommended policy came back grade "β€”" /
8
- overall 0 β†’ the Bug #71 >=70 fitness gate dropped 100% of them β†’ the
9
- recommendation `citations` array was always empty β†’ CitedPolicyCards never
10
- rendered. Meanwhile /api/policies/all scored the SAME policies correctly
11
- (HDFC ERGO Optima Restore = A) because it feeds the FULL curated layer
12
- (main._load_curated_facts, KI-219/251 canonical precedence).
13
-
14
- The fix makes `_scorecard_signal` reuse that EXACT function. This test makes
15
- the two paths *provably* unable to diverge again: for a representative
16
- spread of policies, the recommendation-path GRADE LETTER must equal the
17
- marketplace GRADE LETTER. (Overall scores may differ by a few points
18
- because the marketplace overlays EXTRACTED_DIR data on top; the LETTER β€”
19
- what _recommendation_fit gates on β€” must match.)
 
 
 
 
 
 
 
 
 
 
20
  """
21
  from __future__ import annotations
22
 
@@ -30,59 +40,71 @@ if str(_REPO_ROOT) not in sys.path:
30
 
31
 
32
  class TestScorecardParity(unittest.TestCase):
33
- """Rec-path _scorecard_signal grade letter == marketplace grade letter."""
34
 
35
  @classmethod
36
  def setUpClass(cls):
37
- from backend.main import _load_curated_facts
38
- from backend.scorecard import build_scorecard
39
- from backend.brain_tools import _scorecard_signal, _insurer_reviews
40
 
 
41
  cls._scorecard_signal = staticmethod(_scorecard_signal)
42
- cur = _load_curated_facts()
43
-
44
- # Marketplace grade for a policy_id = build_scorecard on the SAME
45
- # full curated dict + reviews the /api/policies/all path uses.
46
- def _mkt_grade(pid: str):
47
- data = cur.get(pid)
48
- if not data:
49
- return None
50
- slug = data.get("insurer_slug") or (
51
- pid.split("__", 1)[0] if "__" in pid else ""
52
- )
53
- sc = build_scorecard(
54
- data, insurer_reviews=_insurer_reviews(slug), profile=None
55
- )
56
- return sc.grade
57
-
58
- cls._mkt_grade = staticmethod(_mkt_grade)
59
- # A representative spread across grade bands + insurers, incl. the
60
- # user's own policy (HDFC ERGO Optima Restore β€” must be a strong A).
61
- cls._sample = [
62
- "hdfc-ergo__optima-restore",
63
- "hdfc-ergo__my-health-sampoorna-suraksha",
64
- "hdfc-ergo__my-optima-secure",
65
- "oriental-insurance__happy-family-floater",
66
- "star-health__family-health-optima",
67
- ]
68
-
69
- def test_rec_path_grade_matches_marketplace_grade(self):
70
  mismatches = []
71
- for pid in self._sample:
72
- mkt = self._mkt_grade(pid)
73
- if mkt is None:
74
- continue # policy not in curated layer in this env β€” skip
75
- rec = (self._scorecard_signal(pid, profile=None) or {}).get("_grade")
76
- if rec != mkt:
77
- mismatches.append(f"{pid}: rec={rec!r} marketplace={mkt!r}")
 
 
 
 
78
  self.assertEqual(
79
  mismatches, [],
80
- "recommendation-path scorecard grade diverged from the "
81
- "marketplace grade β€” the two paths must feed build_scorecard "
82
- "the SAME curated facts (see KI-FULLFACTS):\n "
83
- + "\n ".join(mismatches),
 
84
  )
85
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  def test_optima_restore_is_a_strong_recommendable_grade(self):
87
  # The user's own policy β€” a genuinely strong plan. It must NOT come
88
  # back as the data-starved "β€”"/0 sentinel, and must clear the Bug
 
1
  """Regression guard β€” recommendation-path scorecard == marketplace scorecard.
2
 
3
+ ROOT CAUSE THIS LOCKS DOWN
4
+ --------------------------
5
+ 1. KI-FULLFACTS (2026-05-17): `brain_tools._scorecard_signal` fed
6
+ `build_scorecard` the 7-key eligibility subset β†’ every recommended
7
+ policy came back grade "β€”"/0 β†’ the >=70 fitness gate dropped 100% β†’
8
+ empty `citations` β†’ CitedPolicyCards never rendered.
9
+
10
+ 2. #40 (2026-05-18): even after (1), `_scorecard_signal` kept its OWN
11
+ doctype-rank + `_merge_curated` re-implementation that *mirrored*
12
+ `/api/policies/all`. Two parallel implementations drift: marketing-
13
+ rename / KI-145 variant / multi-doctype ids graded a different file
14
+ than the marketplace card (e.g. `my:Optima Secure (older variant)`
15
+ recommended at one grade, marketplace card at another).
16
+
17
+ THE FIX β€” SINGLE SOURCE OF TRUTH
18
+ --------------------------------
19
+ `backend.main._marketplace_catalogue()` is the ONE place a card set is
20
+ computed. `backend.main.marketplace_grade(policy_id)` resolves a policy's
21
+ canonical card (UIN-primary) from it. `_scorecard_signal` simply delegates
22
+ to `marketplace_grade`. The two surfaces can no longer diverge because
23
+ there is exactly one computation.
24
+
25
+ This test makes that *provable*: for the ENTIRE id universe the
26
+ recommender can cite β€” every marketplace card id, every curated id, every
27
+ extracted stem incl. `__wordings/__brochure/__cis/__prospectus`
28
+ permutations, and every alias name β€” the recommendation-path GRADE must
29
+ equal the marketplace GRADE. 0 mismatches, by construction.
30
  """
31
  from __future__ import annotations
32
 
 
40
 
41
 
42
  class TestScorecardParity(unittest.TestCase):
43
+ """Rec-path _scorecard_signal grade == marketplace grade, everywhere."""
44
 
45
  @classmethod
46
  def setUpClass(cls):
47
+ import backend.main as M
48
+ from backend.brain_tools import _scorecard_signal
 
49
 
50
+ cls.M = M
51
  cls._scorecard_signal = staticmethod(_scorecard_signal)
52
+ cls._cards = M._marketplace_catalogue(None)
53
+ cls._cur = M._load_curated_facts()
54
+
55
+ def test_full_id_universe_parity(self):
56
+ """The #40 oracle: rec grade == marketplace grade for EVERY id a
57
+ recommendation can surface (card ids + curated ids + extracted
58
+ stems + doctype permutations + aliases). Must be 0 mismatches."""
59
+ M = self.M
60
+ ids = {c.policy_id for c in self._cards}
61
+ ids |= set(self._cur.keys())
62
+ ids |= {p.stem for p in M.settings.EXTRACTED_DIR.glob("*.json")}
63
+ for c in self._cards:
64
+ ids |= set(c.aliases or [])
65
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  mismatches = []
67
+ checked = 0
68
+ for pid in sorted(ids):
69
+ exp = M.marketplace_grade(pid).get("_grade")
70
+ if exp is None:
71
+ continue # not a marketplace card (e.g. regulatory) β€” skip
72
+ checked += 1
73
+ rec = (self._scorecard_signal(pid) or {}).get("_grade")
74
+ if rec != exp:
75
+ mismatches.append(f"{pid}: rec={rec!r} marketplace={exp!r}")
76
+
77
+ self.assertGreater(checked, 140, "id universe collapsed β€” guard broke")
78
  self.assertEqual(
79
  mismatches, [],
80
+ f"#40 PARITY BROKEN ({len(mismatches)}/{checked}): the "
81
+ "recommendation grade diverged from the marketplace grade. "
82
+ "Both MUST flow through backend.main.marketplace_grade (the "
83
+ "single source of truth); do not re-implement scoring in "
84
+ "brain_tools._scorecard_signal.\n " + "\n ".join(mismatches[:40]),
85
  )
86
 
87
+ def test_proven_alias_and_doctype_edges(self):
88
+ """The exact cases #40 used to break: a marketing-rename variant id
89
+ and doctype-permuted ids must each equal their canonical card."""
90
+ M = self.M
91
+ for pid in (
92
+ "hdfc-ergo__my-optima-secure-older-variant",
93
+ "hdfc-ergo__optima-restore__brochure",
94
+ "hdfc-ergo__optima-restore__cis",
95
+ "royal-sundaram__lifeline__cis",
96
+ "new-india__mediclaim-policy__brochure",
97
+ ):
98
+ mkt = M.marketplace_grade(pid).get("_grade")
99
+ if mkt is None:
100
+ continue # absent in this env β€” skip, don't false-fail
101
+ rec = (self._scorecard_signal(pid) or {}).get("_grade")
102
+ self.assertEqual(
103
+ rec, mkt,
104
+ f"{pid}: rec grade {rec!r} != marketplace {mkt!r} "
105
+ "(alias/doctype canonicalisation regressed)",
106
+ )
107
+
108
  def test_optima_restore_is_a_strong_recommendable_grade(self):
109
  # The user's own policy β€” a genuinely strong plan. It must NOT come
110
  # back as the data-starved "β€”"/0 sentinel, and must clear the Bug
tests/test_stt_long_audio_chunking.py ADDED
@@ -0,0 +1,242 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Regression test for KI-302 (2026-05-18) β€” full voice transcript.
2
+
3
+ BUG (user-reported, live): "Full transcript still not coming while
4
+ speaking." A long spoken utterance was truncated to a partial transcript.
5
+
6
+ ROOT CAUSE: Sarvam's saarika REST /speech-to-text endpoint has a hard ~30s
7
+ audio limit. It does NOT 4xx on longer audio β€” it returns HTTP 200 with a
8
+ `transcript` containing ONLY the first ~30s and silently drops the rest.
9
+ The live-voice hook's grace-window batching deliberately merges multiple
10
+ pause-separated speech bursts into ONE blob, so any real-world utterance
11
+ with natural pauses easily exceeds 30s and was silently cut.
12
+
13
+ FIX: `SarvamSTT.transcribe` now decodes the audio, and if it exceeds the
14
+ safe REST ceiling it splits it into <= STT_CHUNK_MS chunks (at silence
15
+ boundaries where possible), transcribes each chunk, and concatenates the
16
+ transcripts in order so the COMPLETE utterance survives.
17
+
18
+ This test pins that contract WITHOUT needing ffmpeg/pydub by stubbing a
19
+ minimal AudioSegment that supports the slicing / len / dBFS / export
20
+ surface the splitter uses, and a fake Sarvam HTTP layer that returns a
21
+ DIFFERENT transcript per 25s window β€” so a regression to single-shot
22
+ behaviour (only the first window transcribed) fails loudly.
23
+
24
+ Run:
25
+ cd /Users/rohitsar/Developer/Insurance\\ Sales\\ Bot
26
+ PYTHONPATH=$PWD .venv/bin/python -m pytest \
27
+ tests/test_stt_long_audio_chunking.py -v
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import asyncio
33
+ import os
34
+ import sys
35
+ import types
36
+
37
+ import pytest
38
+
39
+ os.environ.setdefault("SARVAM_API_KEY", "test-key-for-stt-chunking")
40
+
41
+ from backend.providers.sarvam_stt import ( # noqa: E402
42
+ STT_CHUNK_MS,
43
+ SarvamSTT,
44
+ )
45
+
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # Minimal pydub.AudioSegment stub.
49
+ # The splitter only uses: len(seg) -> ms, seg[a:b] -> sub-segment,
50
+ # seg.dBFS -> float, seg.export(buf, format=...) , set_frame_rate/channels/
51
+ # sample_width (chained no-ops), AudioSegment.from_file(...).
52
+ # Each "ms" carries a sentinel byte so a concatenated/exported chunk can be
53
+ # decoded back into the exact ms range it represents β€” that lets the fake
54
+ # Sarvam server return the words spoken in that window.
55
+ # ---------------------------------------------------------------------------
56
+ class FakeAudioSegment:
57
+ def __init__(self, start_ms: int, end_ms: int):
58
+ self.start_ms = start_ms
59
+ self.end_ms = end_ms # exclusive
60
+
61
+ # --- duration -----------------------------------------------------------
62
+ def __len__(self):
63
+ return self.end_ms - self.start_ms
64
+
65
+ # --- slicing (pydub uses ms-based slicing) ------------------------------
66
+ def __getitem__(self, sl):
67
+ if isinstance(sl, slice):
68
+ lo = 0 if sl.start is None else sl.start
69
+ hi = len(self) if sl.stop is None else sl.stop
70
+ lo = max(0, min(lo, len(self)))
71
+ hi = max(0, min(hi, len(self)))
72
+ return FakeAudioSegment(self.start_ms + lo, self.start_ms + hi)
73
+ raise TypeError("only slice indexing used by splitter")
74
+
75
+ # --- loudness -----------------------------------------------------------
76
+ @property
77
+ def dBFS(self):
78
+ # Constant moderate loudness everywhere EXCEPT a deliberate silent
79
+ # gap at [24000, 24400) ms so _split_on_silence has a real pause to
80
+ # snap to near the 25s ceiling.
81
+ if self.end_ms - self.start_ms == 0:
82
+ return float("-inf")
83
+ if 24000 <= self.start_ms < 24400:
84
+ return float("-inf")
85
+ return -20.0
86
+
87
+ # --- transcode no-ops ---------------------------------------------------
88
+ def set_frame_rate(self, _):
89
+ return self
90
+
91
+ def set_channels(self, _):
92
+ return self
93
+
94
+ def set_sample_width(self, _):
95
+ return self
96
+
97
+ def export(self, buf, format="wav"): # noqa: A002 - pydub signature
98
+ # Encode the ms range so the fake Sarvam server can recover which
99
+ # words this chunk covers.
100
+ buf.write(f"FAKEWAV:{self.start_ms}:{self.end_ms}".encode())
101
+ buf.seek(0)
102
+ return buf
103
+
104
+
105
+ class FakeAudioSegmentFactory:
106
+ @staticmethod
107
+ def from_file(_bio, format=None): # noqa: A002 - pydub signature
108
+ # A 92-second utterance (> 3 Sarvam windows of 25s).
109
+ return FakeAudioSegment(0, 92_000)
110
+
111
+
112
+ @pytest.fixture
113
+ def patch_pydub(monkeypatch):
114
+ fake_pydub = types.ModuleType("pydub")
115
+ fake_pydub.AudioSegment = FakeAudioSegmentFactory
116
+ monkeypatch.setitem(sys.modules, "pydub", fake_pydub)
117
+ yield
118
+
119
+
120
+ # ---------------------------------------------------------------------------
121
+ # Fake Sarvam HTTP layer. Returns the words spoken in the chunk's ms window.
122
+ # The "spoken script" is one word per second: word_0 .. word_91. Sarvam's
123
+ # real 30s truncation is simulated by capping any single call at 30000 ms of
124
+ # audio (it ignores everything past 30s of the blob it receives) β€” exactly
125
+ # the silent-truncation behaviour the fix must defeat by never sending a
126
+ # chunk longer than ~25s.
127
+ # ---------------------------------------------------------------------------
128
+ SARVAM_HARD_LIMIT_MS = 30_000
129
+
130
+
131
+ class _FakeResp:
132
+ def __init__(self, payload):
133
+ self._payload = payload
134
+
135
+ def raise_for_status(self):
136
+ return None
137
+
138
+ def json(self):
139
+ return self._payload
140
+
141
+
142
+ class _FakeClient:
143
+ def __init__(self, *a, **k):
144
+ pass
145
+
146
+ async def __aenter__(self):
147
+ return self
148
+
149
+ async def __aexit__(self, *a):
150
+ return False
151
+
152
+ async def post(self, url, headers=None, files=None, data=None):
153
+ raw = files["file"][1].read()
154
+ text = raw.decode()
155
+ assert text.startswith("FAKEWAV:"), text
156
+ _, s, e = text.split(":")
157
+ start_ms, end_ms = int(s), int(e)
158
+ # Simulate Sarvam's silent 30s truncation on the received blob.
159
+ capped_end = min(end_ms, start_ms + SARVAM_HARD_LIMIT_MS)
160
+ words = [
161
+ f"word_{ms // 1000}"
162
+ for ms in range(start_ms, capped_end, 1000)
163
+ ]
164
+ return _FakeResp({
165
+ "transcript": " ".join(words),
166
+ "language_code": "en-IN",
167
+ "language_probability": 0.99,
168
+ })
169
+
170
+
171
+ @pytest.fixture
172
+ def patch_httpx(monkeypatch):
173
+ import backend.providers.sarvam_stt as mod
174
+
175
+ monkeypatch.setattr(mod.httpx, "AsyncClient", _FakeClient)
176
+ yield
177
+
178
+
179
+ def test_long_utterance_is_fully_transcribed(patch_pydub, patch_httpx):
180
+ """A 92s utterance must yield ALL 92 words, not just the first ~30."""
181
+ stt = SarvamSTT()
182
+ # > 1 KB so the short-audio guard doesn't short-circuit.
183
+ audio = b"\x00" * 4096
184
+ result = asyncio.run(
185
+ stt.transcribe(audio_bytes=audio, audio_format="webm", language_code="en-IN")
186
+ )
187
+
188
+ words = result.text.split()
189
+ # CORE CONTRACT: every one of the 92 spoken words must survive β€” the
190
+ # pre-fix single-shot path returned only word_0..word_29 (Sarvam's 30s
191
+ # silent truncation). A word landing exactly on a silence-snap seam can
192
+ # legitimately appear in BOTH adjoining chunks (a duplicated word at the
193
+ # seam is harmless; a LOST word is the bug). So we assert: (1) no word
194
+ # is missing, (2) order is non-decreasing, (3) the count is ~92 (never
195
+ # ~30). De-duplicating consecutive repeats must reproduce the exact
196
+ # script.
197
+ expected = [f"word_{i}" for i in range(92)]
198
+ deduped = [w for i, w in enumerate(words) if i == 0 or w != words[i - 1]]
199
+ assert deduped == expected, (
200
+ f"transcript truncated/garbled: deduped {len(deduped)} words "
201
+ f"(first={deduped[:3]}, last={deduped[-3:]}), expected 92 in order"
202
+ )
203
+ # Hard proof this is NOT the pre-fix 30s truncation.
204
+ assert len(words) >= 90
205
+ assert "word_91" in words and "word_60" in words
206
+ assert result.raw.get("chunked") is True
207
+ # 92s split at <=25s ceiling => at least 4 chunks.
208
+ assert result.raw.get("chunk_count", 0) >= 4
209
+ assert result.language_code == "en-IN"
210
+
211
+
212
+ def test_short_utterance_single_call_unchanged(patch_pydub, patch_httpx, monkeypatch):
213
+ """A sub-ceiling clip must take exactly ONE Sarvam call (no behaviour
214
+ change / no extra latency for the common case)."""
215
+ # Patch the factory to return a short 8s clip.
216
+ short = FakeAudioSegment(0, 8_000)
217
+ monkeypatch.setattr(
218
+ FakeAudioSegmentFactory, "from_file", staticmethod(lambda *a, **k: short)
219
+ )
220
+ stt = SarvamSTT()
221
+ result = asyncio.run(
222
+ stt.transcribe(audio_bytes=b"\x00" * 4096, audio_format="webm")
223
+ )
224
+ words = result.text.split()
225
+ assert words == [f"word_{i}" for i in range(8)]
226
+ # Single-call path does NOT set the chunked marker.
227
+ assert result.raw.get("chunked") is not True
228
+
229
+
230
+ def test_no_chunk_exceeds_sarvam_safe_ceiling(patch_pydub):
231
+ """Every produced chunk must be <= STT_CHUNK_MS so Sarvam never silently
232
+ truncates a chunk. This is the core invariant the fix rests on."""
233
+ seg = FakeAudioSegment(0, 92_000)
234
+ chunks = SarvamSTT._split_on_silence(seg, STT_CHUNK_MS)
235
+ assert len(chunks) >= 4
236
+ for c in chunks:
237
+ assert len(c) <= STT_CHUNK_MS, f"chunk {len(c)} ms exceeds {STT_CHUNK_MS}"
238
+ # Chunks must tile the whole utterance with no gap and no overlap.
239
+ assert chunks[0].start_ms == 0
240
+ assert chunks[-1].end_ms == 92_000
241
+ for a, b in zip(chunks, chunks[1:]):
242
+ assert a.end_ms == b.start_ms, "chunk boundary gap/overlap loses audio"