SuhasGholkar commited on
Commit
41be979
·
verified ·
1 Parent(s): 3d061cd

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +46 -4
app.py CHANGED
@@ -4396,9 +4396,11 @@ def render_rag_publisher():
4396
  - Columns: chat_id, created_at, user_input, llm_plan (preview)
4397
  Behavior:
4398
  - Uses chat_history.feedback (or feedback_value) = 1
 
4399
  - Do NOT show secured_sql on UI (but include in metadata when pushing)
4400
  - Limit selection to 50 rows per publish
4401
  - Avoid EF conflicts + force ONNX cache into /data/.cache to prevent '/.cache' permission errors.
 
4402
  """
4403
  import streamlit as st, sqlite3, os, datetime, json
4404
 
@@ -4416,6 +4418,17 @@ def render_rag_publisher():
4416
  conn = sqlite3.connect(db_path)
4417
  conn.row_factory = sqlite3.Row
4418
 
 
 
 
 
 
 
 
 
 
 
 
4419
  # Discover schema
4420
  try:
4421
  cols = [r[1] for r in conn.execute("PRAGMA table_info(chat_history)").fetchall()]
@@ -4430,6 +4443,10 @@ def render_rag_publisher():
4430
  fb = "feedback" if "feedback" in cols else ("feedback_value" if "feedback_value" in cols else None)
4431
  role = "user_role" if "user_role" in cols else ("role" if "role" in cols else None)
4432
 
 
 
 
 
4433
  inp_col = "user_input" if "user_input" in cols else ("input_text" if "input_text" in cols else None)
4434
  plan_col = "llm_plan" if "llm_plan" in cols else None
4435
  sql_col = "secured_sql" if "secured_sql" in cols else None # kept for metadata only
@@ -4455,12 +4472,26 @@ def render_rag_publisher():
4455
 
4456
  where = [f"{fb} = 1"]
4457
  params = []
 
 
4458
  if role:
4459
  where.append(f"({role} IS NULL OR {role}='user')")
 
 
4460
  if created:
4461
  where.append(f"{created} BETWEEN ? AND ?")
4462
  params += [f"{start_date} 00:00:00", f"{end_date} 23:59:59"]
4463
 
 
 
 
 
 
 
 
 
 
 
4464
  select_cols = [f"{pk} AS chat_id"]
4465
  select_cols += [f"{created} AS created_at"] if created else ["NULL AS created_at"]
4466
  select_cols += [f"{inp_col} AS user_input"] if inp_col else ["NULL AS user_input"]
@@ -4476,7 +4507,7 @@ def render_rag_publisher():
4476
  {order_by}
4477
  LIMIT ?
4478
  """
4479
- params += [int(max_fetch)]
4480
 
4481
  try:
4482
  rows = conn.execute(sql, params).fetchall()
@@ -4516,11 +4547,11 @@ def render_rag_publisher():
4516
  })
4517
 
4518
  if not data:
4519
- st.info("No positively-rated chats in the selected range.")
4520
  conn.close(); return
4521
 
4522
- df = pd.DataFrame(data)
4523
  st.caption("Select up to 50 records to publish to RAG")
 
4524
  edited = st.data_editor(
4525
  df,
4526
  num_rows="fixed",
@@ -4588,6 +4619,7 @@ def render_rag_publisher():
4588
  cid = int(r["chat_id"])
4589
  uq = (r["user_input"] or "").strip()
4590
  plan_txt = _plan_text(r["llm_plan"]) if "llm_plan" in r.keys() else ""
 
4591
  doc = f"Q: {uq}\nPlan: {plan_txt}"
4592
  docs.append(doc)
4593
  ids.append(f"chat_{cid}")
@@ -4634,7 +4666,17 @@ def render_rag_publisher():
4634
  except Exception: pass
4635
  # -----------------------------------------------------------
4636
 
4637
- # Optional audit log
 
 
 
 
 
 
 
 
 
 
4638
  try:
4639
  conn.execute("""CREATE TABLE IF NOT EXISTS rag_push_log (
4640
  id INTEGER PRIMARY KEY AUTOINCREMENT,
 
4396
  - Columns: chat_id, created_at, user_input, llm_plan (preview)
4397
  Behavior:
4398
  - Uses chat_history.feedback (or feedback_value) = 1
4399
+ - EXCLUDES intents: greeting, thanks, smalltalk, complaint, others
4400
  - Do NOT show secured_sql on UI (but include in metadata when pushing)
4401
  - Limit selection to 50 rows per publish
4402
  - Avoid EF conflicts + force ONNX cache into /data/.cache to prevent '/.cache' permission errors.
4403
+ - Once pushed, a row is hidden from future selections.
4404
  """
4405
  import streamlit as st, sqlite3, os, datetime, json
4406
 
 
4418
  conn = sqlite3.connect(db_path)
4419
  conn.row_factory = sqlite3.Row
4420
 
4421
+ # Track what was already published (so we can hide them)
4422
+ try:
4423
+ conn.execute("""CREATE TABLE IF NOT EXISTS rag_published (
4424
+ chat_id INTEGER PRIMARY KEY,
4425
+ pushed_at TEXT,
4426
+ collection TEXT
4427
+ )""")
4428
+ conn.commit()
4429
+ except Exception:
4430
+ pass
4431
+
4432
  # Discover schema
4433
  try:
4434
  cols = [r[1] for r in conn.execute("PRAGMA table_info(chat_history)").fetchall()]
 
4443
  fb = "feedback" if "feedback" in cols else ("feedback_value" if "feedback_value" in cols else None)
4444
  role = "user_role" if "user_role" in cols else ("role" if "role" in cols else None)
4445
 
4446
+ # Intent column (auto-detect a few common names)
4447
+ intent_col = next((c for c in ["intent", "intent_label", "non_sql_label", "label"] if c in cols), None)
4448
+ excluded_intents = {"greeting", "thanks", "smalltalk", "complaint", "others"}
4449
+
4450
  inp_col = "user_input" if "user_input" in cols else ("input_text" if "input_text" in cols else None)
4451
  plan_col = "llm_plan" if "llm_plan" in cols else None
4452
  sql_col = "secured_sql" if "secured_sql" in cols else None # kept for metadata only
 
4472
 
4473
  where = [f"{fb} = 1"]
4474
  params = []
4475
+
4476
+ # Only end-user messages (if you store role)
4477
  if role:
4478
  where.append(f"({role} IS NULL OR {role}='user')")
4479
+
4480
+ # Date range
4481
  if created:
4482
  where.append(f"{created} BETWEEN ? AND ?")
4483
  params += [f"{start_date} 00:00:00", f"{end_date} 23:59:59"]
4484
 
4485
+ # Exclude published ones
4486
+ where.append(f"{pk} NOT IN (SELECT chat_id FROM rag_published)")
4487
+
4488
+ # Exclude chit-chat intents if we have an intent column
4489
+ intent_params = []
4490
+ if intent_col:
4491
+ placeholders = ",".join(["?"] * len(excluded_intents))
4492
+ where.append(f"LOWER({intent_col}) NOT IN ({placeholders})")
4493
+ intent_params = [s.lower() for s in excluded_intents]
4494
+
4495
  select_cols = [f"{pk} AS chat_id"]
4496
  select_cols += [f"{created} AS created_at"] if created else ["NULL AS created_at"]
4497
  select_cols += [f"{inp_col} AS user_input"] if inp_col else ["NULL AS user_input"]
 
4507
  {order_by}
4508
  LIMIT ?
4509
  """
4510
+ params += intent_params + [int(max_fetch)]
4511
 
4512
  try:
4513
  rows = conn.execute(sql, params).fetchall()
 
4547
  })
4548
 
4549
  if not data:
4550
+ st.info("No eligible positively-rated chats in the selected range.")
4551
  conn.close(); return
4552
 
 
4553
  st.caption("Select up to 50 records to publish to RAG")
4554
+ df = pd.DataFrame(data)
4555
  edited = st.data_editor(
4556
  df,
4557
  num_rows="fixed",
 
4619
  cid = int(r["chat_id"])
4620
  uq = (r["user_input"] or "").strip()
4621
  plan_txt = _plan_text(r["llm_plan"]) if "llm_plan" in r.keys() else ""
4622
+ # Natural-language doc (no SQL in doc text)
4623
  doc = f"Q: {uq}\nPlan: {plan_txt}"
4624
  docs.append(doc)
4625
  ids.append(f"chat_{cid}")
 
4666
  except Exception: pass
4667
  # -----------------------------------------------------------
4668
 
4669
+ # Record each published chat so we don't show it again
4670
+ try:
4671
+ conn.executemany(
4672
+ "INSERT OR IGNORE INTO rag_published (chat_id, pushed_at, collection) VALUES (?, datetime('now'), ?)",
4673
+ [(cid, coll) for cid in selected_ids]
4674
+ )
4675
+ conn.commit()
4676
+ except Exception:
4677
+ pass
4678
+
4679
+ # Optional aggregate audit
4680
  try:
4681
  conn.execute("""CREATE TABLE IF NOT EXISTS rag_push_log (
4682
  id INTEGER PRIMARY KEY AUTOINCREMENT,