SuhasGholkar commited on
Commit
7f59a5b
·
verified ·
1 Parent(s): 3a5b692

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +115 -170
app.py CHANGED
@@ -5206,19 +5206,22 @@ if auth["role"] == "admin":
5206
  st.header("RAG Publisher")
5207
  try:
5208
  reg = schema_registry
5209
- except Exception as e:
5210
  logger.exception("Failed load schema registry : %s", e)
5211
  reg = load_schema_registry(os.path.join(DATA_DIR, SCHEMA_FILE))
5212
  render_rag_publisher()
5213
 
5214
 
5215
- # ===== RAG Publisher (admin) - Positive feedback curation and push to RAG =====
 
5216
  def render_rag_publisher():
 
 
 
 
5217
  import streamlit as st
5218
  import sqlite3, os, datetime
5219
- from typing import List, Dict, Any
5220
 
5221
- # Access control
5222
  role = (st.session_state.get("auth") or {}).get("role")
5223
  if role != "admin":
5224
  st.warning("RAG Publisher is available to admins only.")
@@ -5226,7 +5229,6 @@ def render_rag_publisher():
5226
 
5227
  st.subheader("Publish positively-rated answers to RAG")
5228
 
5229
- # --- DB connection ---
5230
  db_path = os.environ.get("DB_PATH", "/data/olist.sqlite")
5231
  if not os.path.exists(db_path):
5232
  st.error(f"Database not found at {db_path}")
@@ -5234,7 +5236,26 @@ def render_rag_publisher():
5234
  conn = sqlite3.connect(db_path)
5235
  conn.row_factory = sqlite3.Row
5236
 
5237
- # --- Date range inputs (defaults: last 7 days) ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5238
  today = datetime.date.today()
5239
  default_start = today - datetime.timedelta(days=7)
5240
  c1, c2, c3 = st.columns([1,1,1])
@@ -5245,89 +5266,89 @@ def render_rag_publisher():
5245
  with c3:
5246
  st.write(" ")
5247
  st.write(" ")
5248
- limit_input = st.number_input("Max rows to fetch", min_value=1, max_value=500, value=200, step=10, key="rag_max_fetch")
5249
 
5250
  if start_date > end_date:
5251
  st.error("From date cannot be after To date.")
5252
  conn.close()
5253
  return
5254
 
5255
- # --- Fetch positive feedback from normal users in date range ---
5256
- # We try a few likely schemas; fall back gracefully.
5257
- def fetch_pos_feedback(conn, start_date, end_date, limit=200):
5258
- start_iso = f"{start_date} 00:00:00"
5259
- end_iso = f"{end_date} 23:59:59"
5260
- queries = [
5261
- # Common join by chat_id with created_at timestamps
5262
- ("""
5263
- SELECT f.chat_id, f.value AS feedback_value, h.user_text, h.assistant_text, h.created_at, h.user_role
5264
- FROM feedback f
5265
- JOIN chat_history h ON h.chat_id = f.chat_id
5266
- WHERE f.value = 1
5267
- AND (h.user_role IS NULL OR h.user_role = 'user')
5268
- AND (h.created_at BETWEEN ? AND ?)
5269
- ORDER BY h.created_at DESC
5270
- LIMIT ?
5271
- """, (start_iso, end_iso, limit)),
5272
- # Variant: feedback table has ts, chat_history has ts
5273
- ("""
5274
- SELECT f.chat_id, f.value AS feedback_value, h.user_text, h.assistant_text, COALESCE(h.created_at, f.created_at) AS created_at, 'user' AS user_role
5275
- FROM feedback f
5276
- JOIN chat_history h ON h.chat_id = f.chat_id
5277
- WHERE f.value = 1
5278
- AND (COALESCE(h.created_at, f.created_at) BETWEEN ? AND ?)
5279
- ORDER BY created_at DESC
5280
- LIMIT ?
5281
- """, (start_iso, end_iso, limit)),
5282
- # Minimal: if join fails, at least try feedback + last entry for that chat_id
5283
- ("""
5284
- SELECT f.chat_id, f.value AS feedback_value, NULL AS user_text, NULL AS assistant_text, f.created_at AS created_at, 'user' AS user_role
5285
- FROM feedback f
5286
- WHERE f.value = 1
5287
- AND (f.created_at BETWEEN ? AND ?)
5288
- ORDER BY f.created_at DESC
5289
- LIMIT ?
5290
- """, (start_iso, end_iso, limit)),
5291
- ]
5292
- for sql, params in queries:
5293
- try:
5294
- cur = conn.execute(sql, params)
5295
- rows = cur.fetchall()
5296
- if rows:
5297
- return rows
5298
- except Exception:
5299
- continue
5300
- return []
5301
 
5302
- rows = fetch_pos_feedback(conn, start_date, end_date, int(limit_input))
5303
  if not rows:
5304
  st.info("No positively-rated answers found in the selected range.")
5305
  conn.close()
5306
  return
5307
 
5308
- # --- Build selection table (limit final push to 50) ---
 
 
 
 
 
5309
  st.caption("Select up to 50 records to publish to RAG.")
5310
  selected_ids = []
5311
- # display in pages of 25 to avoid too many widgets
5312
  page_size = 25
5313
- pages = (len(rows) + page_size - 1) // page_size
5314
- if pages > 1:
5315
- page = st.number_input("Page", 1, pages, 1)
5316
- else:
5317
- page = 1
5318
  start_idx = (page - 1) * page_size
5319
- end_idx = min(len(rows), start_idx + page_size)
5320
 
5321
- for r in rows[start_idx:end_idx]:
5322
- chat_id = r["chat_id"]
5323
  user_text = (r["user_text"] or "").strip()
5324
  assistant_text = (r["assistant_text"] or "").strip()
5325
- ts = r.get("created_at", "")
5326
- with st.expander(f"chat_id={chat_id} • {ts}"):
 
5327
  st.write("**User**:", user_text if user_text else "_(missing)_")
5328
  st.write("**Assistant**:", assistant_text if assistant_text else "_(missing)_")
5329
- if st.checkbox(f"Select chat_id {chat_id}", key=f"ragpick_{chat_id}"):
5330
- selected_ids.append(int(chat_id))
5331
 
5332
  st.write(f"Selected: **{len(selected_ids)}**")
5333
  if len(selected_ids) > 50:
@@ -5335,7 +5356,6 @@ def render_rag_publisher():
5335
  conn.close()
5336
  return
5337
 
5338
- # --- Publish to RAG section ---
5339
  st.markdown("---")
5340
  st.subheader("Push to RAG")
5341
  rag_col = st.text_input("Collection name", value="rag_feedback", help="Chroma collection to upsert the curated items into.")
@@ -5345,31 +5365,31 @@ def render_rag_publisher():
5345
  st.warning("Please select at least one record.")
5346
  conn.close()
5347
  return
5348
- # Fetch the rows for selected chat_ids
5349
  qmarks = ",".join(["?"] * len(selected_ids))
 
 
 
 
 
 
 
 
 
 
 
5350
  try:
5351
- cur = conn.execute(f"""
5352
- SELECT h.chat_id, h.user_text, h.assistant_text, COALESCE(h.created_at, '') AS created_at
5353
- FROM chat_history h
5354
- WHERE h.chat_id IN ({qmarks})
5355
- ORDER BY h.created_at DESC
5356
- """, selected_ids)
5357
- sel_rows = cur.fetchall()
5358
- except Exception:
5359
- # Fallback: try to at least get assistant_text from a generic table if schema differs
5360
- sel_rows = []
5361
- for cid in selected_ids:
5362
- try:
5363
- cur = conn.execute("SELECT chat_id, user_text, assistant_text, created_at FROM chat_history WHERE chat_id = ? ORDER BY created_at DESC LIMIT 1", (cid,))
5364
- row = cur.fetchone()
5365
- if row: sel_rows.append(row)
5366
- except Exception:
5367
- pass
5368
 
5369
- # Push to Chroma
5370
  try:
5371
  import chromadb
5372
- from chromadb.utils import embedding_functions # if needed by your setup
5373
  client = chromadb.PersistentClient(path=os.environ.get("CHROMA_PATH", "/data/chroma"))
5374
  coll = client.get_or_create_collection(rag_col)
5375
 
@@ -5378,12 +5398,11 @@ def render_rag_publisher():
5378
  cid = row["chat_id"]
5379
  q = (row["user_text"] or "").strip()
5380
  a = (row["assistant_text"] or "").strip()
5381
- created_at = row["created_at"]
5382
- # Combine Q + A as a document; you can tweak this structure
5383
- doc = f"Q: {q}\nA: {a}"
5384
  docs.append(doc)
5385
- ids.append(f"chat_{cid}_{created_at}")
5386
- metas.append({"chat_id": int(cid), "created_at": created_at, "source": "feedback_curated"})
5387
 
5388
  if docs:
5389
  coll.upsert(documents=docs, ids=ids, metadatas=metas)
@@ -5393,12 +5412,9 @@ def render_rag_publisher():
5393
 
5394
  except Exception as e:
5395
  st.error(f"Failed to push to RAG: {e}")
5396
- try:
5397
- import traceback; st.exception(e)
5398
- except Exception:
5399
- pass
5400
 
5401
- # Optional: record a simple push log
5402
  try:
5403
  conn.execute("""CREATE TABLE IF NOT EXISTS rag_push_log (
5404
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -5414,78 +5430,7 @@ def render_rag_publisher():
5414
  except Exception:
5415
  pass
5416
 
5417
- # Done
5418
  conn.close()
5419
  return
5420
 
5421
  conn.close()
5422
- # ===== END RAG Publisher =====
5423
-
5424
- def _render_rag_publisher_inline_debug():
5425
- import os, sys, glob, traceback
5426
- import streamlit as st
5427
-
5428
- st.subheader("RAG Publisher • Diagnostics & Quick-Start")
5429
- cols = st.columns(2)
5430
- with cols[0]:
5431
- auth = st.session_state.get("auth") or {}
5432
- st.write("**Role**:", auth.get("role"))
5433
- st.write("**Working dir**:", os.getcwd())
5434
- st.write("**Python**:", sys.version.split()[0])
5435
- with cols[1]:
5436
- try:
5437
- from importlib.metadata import packages_distributions
5438
- d = packages_distributions()
5439
- st.write("`chromadb` installed:", "chromadb" in d)
5440
- st.write("`sentence-transformers` installed:", "sentence-transformers" in d)
5441
- except Exception:
5442
- st.write("Package metadata unavailable")
5443
-
5444
- # Look for rag_publisher module files on disk for hints
5445
- hits = []
5446
- for pattern in ("rag_publisher.py", "*/rag_publisher/__init__.py", "*/rag_publisher.py"):
5447
- hits.extend(glob.glob(f"/app/{pattern}"))
5448
- hits.extend(glob.glob(f"{os.getcwd()}/{pattern}"))
5449
- if hits:
5450
- st.info("Found potential rag_publisher files:\n" + "\n".join(hits))
5451
- else:
5452
- st.warning("No rag_publisher module found on disk.")
5453
-
5454
- st.markdown("---")
5455
- st.subheader("Minimal Publisher (fallback)")
5456
- st.write("This basic fallback lets you inspect your schema and stage docs so you can finish wiring the real publisher later.")
5457
-
5458
- # Schema preview (if available)
5459
- reg = globals().get("schema_registry")
5460
- if reg:
5461
- try:
5462
- tables = list(reg.keys()) if hasattr(reg, "keys") else []
5463
- with st.expander("Schema Registry (preview)", expanded=False):
5464
- if tables:
5465
- for t in tables:
5466
- st.write(f"- **{t}**: {reg[t]}")
5467
- else:
5468
- st.write("Schema registry present but empty or unrecognized structure.")
5469
- except Exception as e:
5470
- st.error(f"Could not display schema: {e}")
5471
-
5472
- # Staging directory for uploads
5473
- stage_dir = os.path.join(os.environ.get("DATA_DIR", "/data"), "rag_staging")
5474
- os.makedirs(stage_dir, exist_ok=True)
5475
- st.write("**Staging dir:**", stage_dir)
5476
-
5477
- up = st.file_uploader("Upload documents (PDF/TXT/CSV/MD)", type=["pdf","txt","csv","md"], accept_multiple_files=True, key="rag_stage_uploads")
5478
- if up:
5479
- saved = []
5480
- for f in up:
5481
- # sanitize filename
5482
- name = "".join(c for c in f.name if c.isalnum() or c in (" ",".","_","-")).strip()
5483
- dst = os.path.join(stage_dir, name or f"doc_{len(saved)+1}")
5484
- with open(dst, "wb") as fh:
5485
- fh.write(f.getbuffer())
5486
- saved.append(dst)
5487
- st.success(f"Saved {len(saved)} file(s) to {stage_dir}")
5488
- for p in saved:
5489
- st.write("•", p)
5490
-
5491
- st.info("Fallback mode active. To enable full UI, add a `render_rag_publisher()` function in app.py **or** a `rag_publisher` module exposing `render()`.")
 
5206
  st.header("RAG Publisher")
5207
  try:
5208
  reg = schema_registry
5209
+ except Exception:
5210
  logger.exception("Failed load schema registry : %s", e)
5211
  reg = load_schema_registry(os.path.join(DATA_DIR, SCHEMA_FILE))
5212
  render_rag_publisher()
5213
 
5214
 
5215
+
5216
+
5217
  def render_rag_publisher():
5218
+ """
5219
+ Admin tool to curate positively-rated answers and push up to 50 items into a Chroma collection.
5220
+ NOTE: There is NO separate `feedback` table. We only use the `chat_history` table, which contains a `feedback` column.
5221
+ """
5222
  import streamlit as st
5223
  import sqlite3, os, datetime
 
5224
 
 
5225
  role = (st.session_state.get("auth") or {}).get("role")
5226
  if role != "admin":
5227
  st.warning("RAG Publisher is available to admins only.")
 
5229
 
5230
  st.subheader("Publish positively-rated answers to RAG")
5231
 
 
5232
  db_path = os.environ.get("DB_PATH", "/data/olist.sqlite")
5233
  if not os.path.exists(db_path):
5234
  st.error(f"Database not found at {db_path}")
 
5236
  conn = sqlite3.connect(db_path)
5237
  conn.row_factory = sqlite3.Row
5238
 
5239
+ def _chat_cols():
5240
+ try:
5241
+ cur = conn.execute("PRAGMA table_info(chat_history)")
5242
+ rows = cur.fetchall()
5243
+ names = set()
5244
+ for r in rows:
5245
+ try:
5246
+ names.add(r["name"])
5247
+ except Exception:
5248
+ names.add(r[1])
5249
+ return names
5250
+ except Exception:
5251
+ return set()
5252
+
5253
+ cols = _chat_cols()
5254
+ if not cols:
5255
+ st.error("Table `chat_history` not found.")
5256
+ conn.close()
5257
+ return
5258
+
5259
  today = datetime.date.today()
5260
  default_start = today - datetime.timedelta(days=7)
5261
  c1, c2, c3 = st.columns([1,1,1])
 
5266
  with c3:
5267
  st.write(" ")
5268
  st.write(" ")
5269
+ limit_input = st.number_input("Max rows to scan", min_value=10, max_value=1000, value=200, step=10, key="rag_max_fetch")
5270
 
5271
  if start_date > end_date:
5272
  st.error("From date cannot be after To date.")
5273
  conn.close()
5274
  return
5275
 
5276
+ ts_cols = [c for c in ["created_at", "timestamp", "ts", "time", "dt"] if c in cols]
5277
+ ts_col = ts_cols[0] if ts_cols else None
5278
+
5279
+ has_user_text = "user_text" in cols
5280
+ has_assistant_text = "assistant_text" in cols
5281
+ role_col = "user_role" if "user_role" in cols else None
5282
+
5283
+ if "feedback" not in cols:
5284
+ st.error("`chat_history.feedback` column not found. Please add a `feedback` integer column (0/1).")
5285
+ conn.close()
5286
+ return
5287
+
5288
+ where = ["feedback = 1"]
5289
+ params = []
5290
+ if ts_col:
5291
+ where.append(f"{ts_col} BETWEEN ? AND ?")
5292
+ params += [f"{start_date} 00:00:00", f"{end_date} 23:59:59"]
5293
+ if role_col:
5294
+ where.append(f"({role_col} IS NULL OR {role_col} = 'user')")
5295
+
5296
+ select_cols = ["chat_id", "feedback"]
5297
+ select_cols.append("user_text" if has_user_text else "NULL AS user_text")
5298
+ select_cols.append("assistant_text" if has_assistant_text else "NULL AS assistant_text")
5299
+ select_cols.append(f"{ts_col} AS created_at" if ts_col else "NULL AS created_at")
5300
+
5301
+ order_by = f"ORDER BY {ts_col} DESC" if ts_col else "ORDER BY chat_id DESC"
5302
+
5303
+ sql = f\"\"\"
5304
+ SELECT {", ".join(select_cols)}
5305
+ FROM chat_history
5306
+ WHERE {" AND ".join(where)}
5307
+ {order_by}
5308
+ LIMIT ?
5309
+ \"\"\"
5310
+ params += [int(limit_input)]
5311
+
5312
+ try:
5313
+ cur = conn.execute(sql, params)
5314
+ rows = cur.fetchall()
5315
+ except Exception as e:
5316
+ st.error(f"Failed to query chat_history: {e}")
5317
+ try: st.exception(e)
5318
+ except Exception: pass
5319
+ conn.close()
5320
+ return
 
5321
 
 
5322
  if not rows:
5323
  st.info("No positively-rated answers found in the selected range.")
5324
  conn.close()
5325
  return
5326
 
5327
+ seen = set(); dedup = []
5328
+ for r in rows:
5329
+ cid = r["chat_id"]
5330
+ if cid in seen: continue
5331
+ seen.add(cid); dedup.append(r)
5332
+
5333
  st.caption("Select up to 50 records to publish to RAG.")
5334
  selected_ids = []
 
5335
  page_size = 25
5336
+ pages = (len(dedup) + page_size - 1) // page_size
5337
+ page = st.number_input("Page", 1, max(1, pages), 1) if pages > 1 else 1
 
 
 
5338
  start_idx = (page - 1) * page_size
5339
+ end_idx = min(len(dedup), start_idx + page_size)
5340
 
5341
+ for r in dedup[start_idx:end_idx]:
5342
+ cid = r["chat_id"]
5343
  user_text = (r["user_text"] or "").strip()
5344
  assistant_text = (r["assistant_text"] or "").strip()
5345
+ created_at = r["created_at"] if "created_at" in r.keys() else None
5346
+ label = f"chat_id={cid} • {created_at}" if created_at else f"chat_id={cid}"
5347
+ with st.expander(label):
5348
  st.write("**User**:", user_text if user_text else "_(missing)_")
5349
  st.write("**Assistant**:", assistant_text if assistant_text else "_(missing)_")
5350
+ if st.checkbox(f"Select chat_id {cid}", key=f"ragpick_{cid}"):
5351
+ selected_ids.append(int(cid))
5352
 
5353
  st.write(f"Selected: **{len(selected_ids)}**")
5354
  if len(selected_ids) > 50:
 
5356
  conn.close()
5357
  return
5358
 
 
5359
  st.markdown("---")
5360
  st.subheader("Push to RAG")
5361
  rag_col = st.text_input("Collection name", value="rag_feedback", help="Chroma collection to upsert the curated items into.")
 
5365
  st.warning("Please select at least one record.")
5366
  conn.close()
5367
  return
5368
+
5369
  qmarks = ",".join(["?"] * len(selected_ids))
5370
+ select_cols2 = ["chat_id"]
5371
+ select_cols2 += ["user_text" if has_user_text else "NULL AS user_text"]
5372
+ select_cols2 += ["assistant_text" if has_assistant_text else "NULL AS assistant_text"]
5373
+ select_cols2 += [f"{ts_col} AS created_at" if ts_col else "NULL AS created_at"]
5374
+
5375
+ sql2 = f\"\"\"
5376
+ SELECT {", ".join(select_cols2)}
5377
+ FROM chat_history
5378
+ WHERE chat_id IN ({qmarks})
5379
+ {order_by}
5380
+ \"\"\"
5381
  try:
5382
+ cur2 = conn.execute(sql2, selected_ids)
5383
+ sel_rows = cur2.fetchall()
5384
+ except Exception as e:
5385
+ st.error(f"Failed to fetch selected rows: {e}")
5386
+ try: st.exception(e)
5387
+ except Exception: pass
5388
+ conn.close()
5389
+ return
 
 
 
 
 
 
 
 
 
5390
 
 
5391
  try:
5392
  import chromadb
 
5393
  client = chromadb.PersistentClient(path=os.environ.get("CHROMA_PATH", "/data/chroma"))
5394
  coll = client.get_or_create_collection(rag_col)
5395
 
 
5398
  cid = row["chat_id"]
5399
  q = (row["user_text"] or "").strip()
5400
  a = (row["assistant_text"] or "").strip()
5401
+ tsv = row["created_at"] if "created_at" in row.keys() else ""
5402
+ doc = f"Q: {q}\\nA: {a}"
 
5403
  docs.append(doc)
5404
+ ids.append(f"chat_{cid}_{tsv}")
5405
+ metas.append({"chat_id": int(cid), "created_at": tsv, "source": "feedback_curated"})
5406
 
5407
  if docs:
5408
  coll.upsert(documents=docs, ids=ids, metadatas=metas)
 
5412
 
5413
  except Exception as e:
5414
  st.error(f"Failed to push to RAG: {e}")
5415
+ try: st.exception(e)
5416
+ except Exception: pass
 
 
5417
 
 
5418
  try:
5419
  conn.execute("""CREATE TABLE IF NOT EXISTS rag_push_log (
5420
  id INTEGER PRIMARY KEY AUTOINCREMENT,
 
5430
  except Exception:
5431
  pass
5432
 
 
5433
  conn.close()
5434
  return
5435
 
5436
  conn.close()