Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,645 +1,217 @@
|
|
| 1 |
-
# app.py
|
| 2 |
-
|
| 3 |
import streamlit as st
|
| 4 |
import os
|
| 5 |
from pathlib import Path
|
| 6 |
from typing import Dict, List, Optional, Any
|
| 7 |
from datetime import datetime
|
| 8 |
-
from threading import Lock
|
| 9 |
from dataclasses import dataclass
|
| 10 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 11 |
from langchain.vectorstores import FAISS
|
| 12 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 13 |
from langchain_community.chat_models import ChatOpenAI
|
| 14 |
-
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
| 15 |
-
from langchain_core.messages import HumanMessage, AIMessage, SystemMessage
|
| 16 |
from langchain.chains import ConversationalRetrievalChain
|
| 17 |
-
from langchain_core.runnables import RunnablePassthrough
|
| 18 |
-
from langchain.memory import ConversationBufferMemory
|
| 19 |
|
| 20 |
from utils.database import (
|
| 21 |
-
# Base database operations
|
| 22 |
create_connection,
|
| 23 |
create_tables,
|
| 24 |
get_all_documents,
|
| 25 |
-
force_recreate_collections_tables,
|
| 26 |
-
|
| 27 |
-
# Collection operations
|
| 28 |
get_collections,
|
| 29 |
-
create_collection,
|
| 30 |
-
add_document_to_collection,
|
| 31 |
-
remove_from_collection,
|
| 32 |
-
update_collection,
|
| 33 |
get_collection_documents,
|
| 34 |
-
handle_document_upload,
|
| 35 |
-
verify_database_tables,
|
| 36 |
initialize_qa_system,
|
| 37 |
get_embeddings_model,
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
)
|
| 41 |
-
|
| 42 |
-
# Component imports
|
| 43 |
-
from components.chat import display_chat_interface
|
| 44 |
-
from components.collection_manager import (
|
| 45 |
-
display_enhanced_collections,
|
| 46 |
-
show_collection_creation_dialog
|
| 47 |
)
|
| 48 |
|
| 49 |
-
# Optional analytics import if you're using it
|
| 50 |
-
from utils.analytics import display_analytics_dashboard
|
| 51 |
-
|
| 52 |
-
from components.chat import display_chat_interface
|
| 53 |
-
from components.document_store import display_documents_tab
|
| 54 |
-
|
| 55 |
-
# Create locks for thread-safe operations
|
| 56 |
-
conn_lock = Lock()
|
| 57 |
-
if not os.path.exists('/data'):
|
| 58 |
-
try:
|
| 59 |
-
from setup import setup_directories
|
| 60 |
-
if not setup_directories():
|
| 61 |
-
raise Exception("Failed to set up directories")
|
| 62 |
-
except Exception as e:
|
| 63 |
-
st.error(f"Setup error: {e}")
|
| 64 |
-
st.stop()
|
| 65 |
-
|
| 66 |
-
def handle_chat_state_change():
|
| 67 |
-
"""Handle changes in chat state when switching tabs or collections."""
|
| 68 |
-
if st.session_state.get('chat_ready'):
|
| 69 |
-
# Check if we need to reinitialize with new documents
|
| 70 |
-
if st.session_state.get('reinitialize_chat'):
|
| 71 |
-
initialize_chat_from_collection()
|
| 72 |
-
st.session_state.reinitialize_chat = False
|
| 73 |
-
|
| 74 |
-
def verify_database_tables(conn):
|
| 75 |
-
"""Verify that all required tables exist."""
|
| 76 |
-
try:
|
| 77 |
-
cursor = conn.cursor()
|
| 78 |
-
# Get list of all tables
|
| 79 |
-
cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
| 80 |
-
tables = [table[0] for table in cursor.fetchall()]
|
| 81 |
-
|
| 82 |
-
# Check for required tables
|
| 83 |
-
required_tables = {
|
| 84 |
-
'documents', 'queries', 'annotations',
|
| 85 |
-
'collections', 'document_collections', 'chats', 'chat_messages'
|
| 86 |
-
}
|
| 87 |
-
|
| 88 |
-
missing_tables = required_tables - set(tables)
|
| 89 |
-
|
| 90 |
-
# If collections table doesn't exist, force recreate it
|
| 91 |
-
if 'collections' not in tables:
|
| 92 |
-
if force_recreate_collections_tables(conn):
|
| 93 |
-
st.session_state.show_debug = True
|
| 94 |
-
else:
|
| 95 |
-
st.error("Failed to create required database tables.")
|
| 96 |
-
return False
|
| 97 |
-
|
| 98 |
-
return True
|
| 99 |
-
|
| 100 |
-
except Exception as e:
|
| 101 |
-
st.error(f"Error verifying database tables: {e}")
|
| 102 |
-
return False
|
| 103 |
-
|
| 104 |
-
def initialize_database():
|
| 105 |
-
"""Initialize database with persistent storage."""
|
| 106 |
-
try:
|
| 107 |
-
if 'db_conn' not in st.session_state:
|
| 108 |
-
# Get storage path
|
| 109 |
-
if os.path.exists('/data'):
|
| 110 |
-
base_path = Path('/data')
|
| 111 |
-
else:
|
| 112 |
-
base_path = Path(os.getcwd()) / 'data'
|
| 113 |
-
|
| 114 |
-
# Ensure data directory exists
|
| 115 |
-
base_path.mkdir(parents=True, exist_ok=True)
|
| 116 |
-
|
| 117 |
-
# Create database path
|
| 118 |
-
db_path = base_path / 'rfp_analysis.db'
|
| 119 |
-
|
| 120 |
-
try:
|
| 121 |
-
# Ensure file can be created
|
| 122 |
-
db_path.touch(exist_ok=True)
|
| 123 |
-
except Exception as e:
|
| 124 |
-
st.error(f"Failed to create database file: {e}")
|
| 125 |
-
return False
|
| 126 |
-
|
| 127 |
-
# Create connection
|
| 128 |
-
conn = create_connection(str(db_path))
|
| 129 |
-
if conn is not None:
|
| 130 |
-
# Create all tables (includes collection tables)
|
| 131 |
-
create_tables(conn)
|
| 132 |
-
st.session_state.db_conn = conn
|
| 133 |
-
|
| 134 |
-
# Verify tables were created
|
| 135 |
-
verify_database_tables(conn)
|
| 136 |
-
return True
|
| 137 |
-
else:
|
| 138 |
-
return False
|
| 139 |
-
else:
|
| 140 |
-
# Verify tables exist in existing connection
|
| 141 |
-
verify_database_tables(st.session_state.db_conn)
|
| 142 |
-
return True
|
| 143 |
-
|
| 144 |
-
except Exception as e:
|
| 145 |
-
st.error(f"Database initialization error: {e}")
|
| 146 |
-
return False
|
| 147 |
-
|
| 148 |
@dataclass
|
| 149 |
-
class
|
| 150 |
-
"""
|
| 151 |
-
# Collection Management
|
| 152 |
show_collection_dialog: bool = False
|
| 153 |
selected_collection: Optional[Dict] = None
|
| 154 |
-
current_collection: Optional[Dict] = None
|
| 155 |
-
|
| 156 |
-
# Chat System
|
| 157 |
chat_ready: bool = False
|
| 158 |
-
messages:
|
| 159 |
current_chat_id: Optional[int] = None
|
| 160 |
-
|
| 161 |
-
# Document Processing
|
| 162 |
vector_store: Optional[Any] = None
|
| 163 |
qa_system: Optional[Any] = None
|
| 164 |
-
|
| 165 |
-
# State Management
|
| 166 |
reinitialize_chat: bool = False
|
| 167 |
-
debug_mode: bool = False
|
| 168 |
-
|
| 169 |
-
def initialize_session_state() -> None:
|
| 170 |
-
"""
|
| 171 |
-
Initialize all session state variables with proper typing and documentation.
|
| 172 |
-
"""
|
| 173 |
-
defaults = SessionStateDefaults()
|
| 174 |
-
|
| 175 |
-
# Dictionary mapping state variables to their descriptions
|
| 176 |
-
state_vars: Dict[str, Dict[str, Any]] = {
|
| 177 |
-
'show_collection_dialog': {
|
| 178 |
-
'default': defaults.show_collection_dialog,
|
| 179 |
-
'desc': 'Controls visibility of collection creation dialog'
|
| 180 |
-
},
|
| 181 |
-
'selected_collection': {
|
| 182 |
-
'default': defaults.selected_collection,
|
| 183 |
-
'desc': 'Currently selected collection for viewing/editing'
|
| 184 |
-
},
|
| 185 |
-
'current_collection': {
|
| 186 |
-
'default': defaults.current_collection,
|
| 187 |
-
'desc': 'Active collection for chat/analysis'
|
| 188 |
-
},
|
| 189 |
-
'chat_ready': {
|
| 190 |
-
'default': defaults.chat_ready,
|
| 191 |
-
'desc': 'Indicates if chat system is initialized and ready'
|
| 192 |
-
},
|
| 193 |
-
'messages': {
|
| 194 |
-
'default': [] if defaults.messages is None else defaults.messages,
|
| 195 |
-
'desc': 'List of chat messages in the current session'
|
| 196 |
-
},
|
| 197 |
-
'current_chat_id': {
|
| 198 |
-
'default': defaults.current_chat_id,
|
| 199 |
-
'desc': 'ID of the current chat session'
|
| 200 |
-
},
|
| 201 |
-
'vector_store': {
|
| 202 |
-
'default': defaults.vector_store,
|
| 203 |
-
'desc': 'FAISS vector store for document embeddings'
|
| 204 |
-
},
|
| 205 |
-
'qa_system': {
|
| 206 |
-
'default': defaults.qa_system,
|
| 207 |
-
'desc': 'Initialized QA system for chat'
|
| 208 |
-
},
|
| 209 |
-
'reinitialize_chat': {
|
| 210 |
-
'default': defaults.reinitialize_chat,
|
| 211 |
-
'desc': 'Flag to trigger chat system reinitialization'
|
| 212 |
-
},
|
| 213 |
-
'debug_mode': {
|
| 214 |
-
'default': defaults.debug_mode,
|
| 215 |
-
'desc': 'Enable/disable debug information'
|
| 216 |
-
}
|
| 217 |
-
}
|
| 218 |
-
|
| 219 |
-
# Initialize each state variable if not already present
|
| 220 |
-
for var_name, config in state_vars.items():
|
| 221 |
-
if var_name not in st.session_state:
|
| 222 |
-
st.session_state[var_name] = config['default']
|
| 223 |
|
| 224 |
-
def
|
| 225 |
-
"""
|
| 226 |
-
|
| 227 |
-
st.session_state
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
"""
|
| 242 |
-
return {
|
| 243 |
-
key: value for key, value in st.session_state.items()
|
| 244 |
-
if not key.startswith('_') # Exclude internal Streamlit states
|
| 245 |
-
}
|
| 246 |
-
|
| 247 |
-
def display_top_bar():
|
| 248 |
-
"""Display the application's top navigation bar."""
|
| 249 |
-
col1, col2, col3 = st.columns([1, 3, 1])
|
| 250 |
-
|
| 251 |
-
with col1:
|
| 252 |
-
if os.path.exists("img/logo.png"):
|
| 253 |
-
st.image("img/logo.png", width=100)
|
| 254 |
-
else:
|
| 255 |
-
st.warning("Logo not found at img/logo.png")
|
| 256 |
-
|
| 257 |
-
with col2:
|
| 258 |
-
st.title("Synaptyx.AI - RFP Analysis Agent")
|
| 259 |
-
|
| 260 |
-
with col3:
|
| 261 |
-
if st.session_state.current_collection:
|
| 262 |
-
st.info(f"๐ Active Collection: {st.session_state.current_collection['name']}")
|
| 263 |
-
|
| 264 |
-
def initialize_chat_from_existing():
|
| 265 |
-
"""Initialize chat system from existing documents in database."""
|
| 266 |
-
if not st.session_state.get('chat_ready'):
|
| 267 |
-
try:
|
| 268 |
-
documents = get_all_documents(st.session_state.db_conn)
|
| 269 |
-
if documents:
|
| 270 |
-
# Initialize vector store and QA system with existing documents
|
| 271 |
-
embeddings = get_embeddings_model()
|
| 272 |
-
chunks = []
|
| 273 |
-
for doc in documents:
|
| 274 |
-
doc_chunks = text_splitter.split_text(doc['content'])
|
| 275 |
-
for chunk in doc_chunks:
|
| 276 |
-
chunks.append({
|
| 277 |
-
'content': chunk,
|
| 278 |
-
'metadata': {'source': doc['name'], 'document_id': doc['id']}
|
| 279 |
-
})
|
| 280 |
-
|
| 281 |
-
vector_store = FAISS.from_texts(
|
| 282 |
-
[chunk['content'] for chunk in chunks],
|
| 283 |
-
embeddings,
|
| 284 |
-
[chunk['metadata'] for chunk in chunks]
|
| 285 |
-
)
|
| 286 |
-
|
| 287 |
-
st.session_state.vector_store = vector_store
|
| 288 |
-
st.session_state.qa_system = initialize_qa_system(vector_store)
|
| 289 |
-
st.session_state.chat_ready = True
|
| 290 |
-
return True
|
| 291 |
-
except Exception as e:
|
| 292 |
-
st.error(f"Error initializing chat: {e}")
|
| 293 |
-
return False
|
| 294 |
-
|
| 295 |
-
def initialize_chat_from_collection():
|
| 296 |
-
"""Initialize chat system with vector store reuse."""
|
| 297 |
try:
|
| 298 |
-
documents
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
documents
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
|
|
|
|
|
|
| 312 |
# Reuse existing vector store
|
| 313 |
-
st.session_state.
|
| 314 |
-
st.session_state.
|
| 315 |
-
return True
|
| 316 |
-
|
| 317 |
-
# If no existing vector store, create new one
|
| 318 |
-
with st.spinner("Initializing chat system..."):
|
| 319 |
-
embeddings = get_embeddings_model()
|
| 320 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
| 321 |
-
chunk_size=500,
|
| 322 |
-
chunk_overlap=50,
|
| 323 |
-
length_function=len,
|
| 324 |
-
separators=["\n\n", "\n", " ", ""]
|
| 325 |
-
)
|
| 326 |
-
|
| 327 |
-
chunks = []
|
| 328 |
-
for doc in documents:
|
| 329 |
-
doc_chunks = text_splitter.split_text(doc['content'])
|
| 330 |
-
for chunk in doc_chunks:
|
| 331 |
-
chunks.append({
|
| 332 |
-
'content': chunk,
|
| 333 |
-
'metadata': {'source': doc['name'], 'document_id': doc['id']}
|
| 334 |
-
})
|
| 335 |
-
|
| 336 |
-
vector_store = FAISS.from_texts(
|
| 337 |
-
[chunk['content'] for chunk in chunks],
|
| 338 |
-
embeddings,
|
| 339 |
-
[chunk['metadata'] for chunk in chunks]
|
| 340 |
-
)
|
| 341 |
-
|
| 342 |
-
st.session_state.vector_store = vector_store
|
| 343 |
-
st.session_state.qa_system = initialize_qa_system(vector_store)
|
| 344 |
return True
|
| 345 |
-
|
| 346 |
-
return False
|
| 347 |
-
|
| 348 |
-
except Exception as e:
|
| 349 |
-
st.error(f"Error initializing chat: {e}")
|
| 350 |
-
return False
|
| 351 |
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
st.session_state.current_chat_id = None
|
| 360 |
-
if st.session_state.chat_ready:
|
| 361 |
-
st.rerun()
|
| 362 |
-
|
| 363 |
-
st.divider()
|
| 364 |
-
|
| 365 |
-
# Collection Management
|
| 366 |
-
st.title("๐ Collections")
|
| 367 |
-
collections = get_collections(st.session_state.db_conn)
|
| 368 |
-
|
| 369 |
-
if st.button("โ Create Collection", use_container_width=True):
|
| 370 |
-
st.session_state.show_collection_dialog = True
|
| 371 |
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
st.caption(f"Uploaded: {doc['upload_date']}")
|
| 389 |
-
if st.button("Use for Chat", key=f"use_{doc['id']}"):
|
| 390 |
-
initialize_chat_for_document(doc['id'])
|
| 391 |
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
with col1:
|
| 401 |
-
if st.button("Create", key="sidebar_create_btn"):
|
| 402 |
-
if name:
|
| 403 |
-
if create_collection(st.session_state.db_conn, name, description):
|
| 404 |
-
st.success(f"Collection '{name}' created!")
|
| 405 |
-
st.session_state.show_collection_dialog = False
|
| 406 |
-
st.rerun()
|
| 407 |
-
else:
|
| 408 |
-
st.error("Please enter a collection name")
|
| 409 |
-
|
| 410 |
-
with col2:
|
| 411 |
-
if st.button("Cancel", key="sidebar_cancel_btn"):
|
| 412 |
-
st.session_state.show_collection_dialog = False
|
| 413 |
-
st.rerun()
|
| 414 |
|
| 415 |
def display_welcome_screen():
|
| 416 |
-
"""Display
|
| 417 |
-
st.title("๐ค Welcome to
|
| 418 |
-
st.markdown("### Your AI-powered RFP Analysis Assistant")
|
| 419 |
-
|
| 420 |
-
# Check for existing documents
|
| 421 |
-
documents = get_all_documents(st.session_state.db_conn)
|
| 422 |
-
collections = get_collections(st.session_state.db_conn)
|
| 423 |
|
| 424 |
col1, col2 = st.columns(2)
|
| 425 |
with col1:
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
1. Upload your RFP documents using the sidebar
|
| 429 |
-
2. Organize documents into collections
|
| 430 |
-
3. Start analyzing with AI!
|
| 431 |
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
|
|
|
| 438 |
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
st.rerun()
|
| 448 |
-
with col_b:
|
| 449 |
-
if st.button("View Documents", use_container_width=True):
|
| 450 |
-
st.session_state.show_document_store = True
|
| 451 |
-
st.rerun()
|
| 452 |
|
| 453 |
with col2:
|
| 454 |
-
st.
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
"
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
"๐ Find similar sections across RFPs"
|
| 463 |
-
]
|
| 464 |
-
for example in examples:
|
| 465 |
-
st.markdown(f"โข {example}")
|
| 466 |
-
|
| 467 |
-
# Add collection stats if they exist
|
| 468 |
-
if collections:
|
| 469 |
-
st.markdown("---")
|
| 470 |
-
st.markdown("#### Your Collections:")
|
| 471 |
-
for collection in collections[:3]: # Show top 3 collections
|
| 472 |
-
with st.container():
|
| 473 |
-
st.markdown(f"""
|
| 474 |
-
๐๏ธ **{collection['name']}**
|
| 475 |
-
Documents: {collection['doc_count']}
|
| 476 |
-
""")
|
| 477 |
-
if len(collections) > 3:
|
| 478 |
-
st.caption("... and more collections")
|
| 479 |
-
|
| 480 |
-
def display_chat_area():
|
| 481 |
-
"""Display the main chat interface or welcome screen with proper state management."""
|
| 482 |
-
if not st.session_state.chat_ready:
|
| 483 |
-
display_welcome_screen()
|
| 484 |
-
else:
|
| 485 |
-
col1, col2 = st.columns([3, 1])
|
| 486 |
-
with col1:
|
| 487 |
-
display_chat_interface()
|
| 488 |
-
with col2:
|
| 489 |
-
# Document context panel
|
| 490 |
-
st.markdown("### Current Context")
|
| 491 |
-
if st.session_state.current_collection:
|
| 492 |
-
st.info(f"๐ Using Collection: {st.session_state.current_collection['name']}")
|
| 493 |
-
else:
|
| 494 |
-
st.info("๐ Using All Documents")
|
| 495 |
-
|
| 496 |
-
# Quick actions
|
| 497 |
-
st.markdown("### Quick Actions")
|
| 498 |
-
if st.button("๐ New Chat", use_container_width=True):
|
| 499 |
-
reset_chat_state()
|
| 500 |
-
st.rerun()
|
| 501 |
-
if st.button("๐ Export Chat", use_container_width=True):
|
| 502 |
-
export_chat_history()
|
| 503 |
-
|
| 504 |
-
# Example questions for quick reference
|
| 505 |
-
with st.expander("๐ก Question Examples", expanded=False):
|
| 506 |
-
examples = [
|
| 507 |
-
"Summarize main points",
|
| 508 |
-
"Extract requirements",
|
| 509 |
-
"Compare solutions",
|
| 510 |
-
"Analyze pricing"
|
| 511 |
-
]
|
| 512 |
-
for example in examples:
|
| 513 |
-
if st.button(example, key=f"example_{example}"):
|
| 514 |
-
# Set the example as the current question
|
| 515 |
-
st.session_state.current_question = example
|
| 516 |
st.rerun()
|
| 517 |
|
| 518 |
-
def
|
| 519 |
-
"""
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 531 |
|
| 532 |
def main():
|
| 533 |
-
"""Main application function."""
|
| 534 |
-
# Page configuration
|
| 535 |
st.set_page_config(
|
|
|
|
| 536 |
layout="wide",
|
| 537 |
-
page_title="SYNAPTYX - RFP Analysis Agent",
|
| 538 |
initial_sidebar_state="expanded"
|
| 539 |
)
|
| 540 |
|
| 541 |
-
# Initialize
|
| 542 |
-
if not initialize_database():
|
| 543 |
-
st.error("Failed to initialize database. Please contact support.")
|
| 544 |
-
return
|
| 545 |
-
|
| 546 |
initialize_session_state()
|
| 547 |
|
| 548 |
-
#
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
with col2:
|
| 556 |
-
st.title("SYNAPTYX - RFP Analysis Agent")
|
| 557 |
-
with col3:
|
| 558 |
-
if st.session_state.current_collection:
|
| 559 |
-
st.info(f"๐ Active Collection: {st.session_state.current_collection['name']}")
|
| 560 |
|
| 561 |
-
# Sidebar
|
| 562 |
with st.sidebar:
|
| 563 |
-
|
| 564 |
-
st.title("๐ฌ Chat Controls")
|
| 565 |
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
reset_chat_state()
|
| 569 |
st.rerun()
|
| 570 |
|
| 571 |
-
st.
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
# Collection Creation Button
|
| 577 |
-
if st.button("โ Create New Collection", use_container_width=True):
|
| 578 |
-
st.session_state.show_collection_dialog = True
|
| 579 |
-
|
| 580 |
-
# Collection Selection
|
| 581 |
-
collections = get_collections(st.session_state.db_conn)
|
| 582 |
-
if collections:
|
| 583 |
-
selected = st.selectbox(
|
| 584 |
-
"Select Collection",
|
| 585 |
-
options=["All Documents"] + [c['name'] for c in collections],
|
| 586 |
-
key="collection_select"
|
| 587 |
-
)
|
| 588 |
-
|
| 589 |
-
if selected != "All Documents":
|
| 590 |
-
collection = next((c for c in collections if c['name'] == selected), None)
|
| 591 |
-
if collection:
|
| 592 |
-
st.session_state.current_collection = collection
|
| 593 |
-
display_collection_documents(collection['id'])
|
| 594 |
-
|
| 595 |
-
# Upload Section
|
| 596 |
-
st.header("Upload Documents")
|
| 597 |
-
uploaded_files = st.file_uploader(
|
| 598 |
-
"Upload PDF documents",
|
| 599 |
-
type=['pdf'],
|
| 600 |
-
accept_multiple_files=True,
|
| 601 |
-
help="Limit 200MB per file โข PDF"
|
| 602 |
-
)
|
| 603 |
-
|
| 604 |
-
if uploaded_files:
|
| 605 |
-
handle_document_upload(uploaded_files,
|
| 606 |
-
collection_id=st.session_state.current_collection.get('id') if st.session_state.current_collection else None)
|
| 607 |
-
|
| 608 |
-
# Show collection creation dialog if triggered
|
| 609 |
-
if st.session_state.show_collection_dialog:
|
| 610 |
-
show_collection_creation_dialog()
|
| 611 |
|
| 612 |
# Main content area
|
| 613 |
-
if st.session_state.chat_ready:
|
| 614 |
-
display_chat_interface()
|
| 615 |
-
else:
|
| 616 |
display_welcome_screen()
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
if st.session_state.debug_mode and st.sidebar.checkbox("Show Debug Info"):
|
| 620 |
-
st.sidebar.write("Current State:", get_current_state())
|
| 621 |
-
|
| 622 |
-
def display_collection_documents(collection_id: int):
|
| 623 |
-
"""Display documents in the selected collection."""
|
| 624 |
-
documents = get_collection_documents(st.session_state.db_conn, collection_id)
|
| 625 |
-
if documents:
|
| 626 |
-
st.markdown("### Documents in Collection")
|
| 627 |
-
for doc in documents:
|
| 628 |
-
with st.expander(f"๐ {doc['name']}", expanded=False):
|
| 629 |
-
st.caption(f"Uploaded: {doc['upload_date']}")
|
| 630 |
-
col1, col2 = st.columns(2)
|
| 631 |
-
with col1:
|
| 632 |
-
if st.button("Preview", key=f"preview_{doc['id']}"):
|
| 633 |
-
st.text_area(
|
| 634 |
-
"Content Preview",
|
| 635 |
-
value=doc['content'][:500] + "..." if len(doc['content']) > 500 else doc['content'],
|
| 636 |
-
height=100,
|
| 637 |
-
disabled=True
|
| 638 |
-
)
|
| 639 |
-
with col2:
|
| 640 |
-
if st.button("Use for Chat", key=f"chat_{doc['id']}"):
|
| 641 |
-
initialize_chat_from_collection()
|
| 642 |
-
st.rerun()
|
| 643 |
|
| 644 |
if __name__ == "__main__":
|
| 645 |
main()
|
|
|
|
|
|
|
|
|
|
| 1 |
import streamlit as st
|
| 2 |
import os
|
| 3 |
from pathlib import Path
|
| 4 |
from typing import Dict, List, Optional, Any
|
| 5 |
from datetime import datetime
|
|
|
|
| 6 |
from dataclasses import dataclass
|
| 7 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 8 |
from langchain.vectorstores import FAISS
|
| 9 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 10 |
from langchain_community.chat_models import ChatOpenAI
|
| 11 |
+
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
| 12 |
+
from langchain_core.messages import HumanMessage, AIMessage, SystemMessage
|
| 13 |
from langchain.chains import ConversationalRetrievalChain
|
|
|
|
|
|
|
| 14 |
|
| 15 |
from utils.database import (
|
|
|
|
| 16 |
create_connection,
|
| 17 |
create_tables,
|
| 18 |
get_all_documents,
|
|
|
|
|
|
|
|
|
|
| 19 |
get_collections,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
get_collection_documents,
|
|
|
|
|
|
|
| 21 |
initialize_qa_system,
|
| 22 |
get_embeddings_model,
|
| 23 |
+
verify_database_tables,
|
| 24 |
+
handle_document_upload
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
)
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
@dataclass
|
| 28 |
+
class SessionState:
|
| 29 |
+
"""Centralized session state management."""
|
|
|
|
| 30 |
show_collection_dialog: bool = False
|
| 31 |
selected_collection: Optional[Dict] = None
|
|
|
|
|
|
|
|
|
|
| 32 |
chat_ready: bool = False
|
| 33 |
+
messages: List = None
|
| 34 |
current_chat_id: Optional[int] = None
|
|
|
|
|
|
|
| 35 |
vector_store: Optional[Any] = None
|
| 36 |
qa_system: Optional[Any] = None
|
|
|
|
|
|
|
| 37 |
reinitialize_chat: bool = False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
+
def initialize_session_state():
|
| 40 |
+
"""Initialize session state with default values."""
|
| 41 |
+
defaults = SessionState()
|
| 42 |
+
if 'initialized' not in st.session_state:
|
| 43 |
+
st.session_state.update({
|
| 44 |
+
'show_collection_dialog': defaults.show_collection_dialog,
|
| 45 |
+
'selected_collection': defaults.selected_collection,
|
| 46 |
+
'chat_ready': defaults.chat_ready,
|
| 47 |
+
'messages': [] if defaults.messages is None else defaults.messages,
|
| 48 |
+
'current_chat_id': defaults.current_chat_id,
|
| 49 |
+
'vector_store': defaults.vector_store,
|
| 50 |
+
'qa_system': defaults.qa_system,
|
| 51 |
+
'reinitialize_chat': defaults.reinitialize_chat,
|
| 52 |
+
'initialized': True
|
| 53 |
+
})
|
| 54 |
+
|
| 55 |
+
def initialize_chat_system(collection_id: Optional[int] = None) -> bool:
|
| 56 |
+
"""Initialize chat system with optimized document loading."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
try:
|
| 58 |
+
# Get documents based on collection or all documents
|
| 59 |
+
documents = (get_collection_documents(st.session_state.db_conn, collection_id)
|
| 60 |
+
if collection_id else get_all_documents(st.session_state.db_conn))
|
| 61 |
+
|
| 62 |
+
if not documents:
|
| 63 |
+
st.error("No documents found.")
|
| 64 |
+
return False
|
| 65 |
+
|
| 66 |
+
# Check for existing vector store with same documents
|
| 67 |
+
document_ids = [doc['id'] for doc in documents]
|
| 68 |
+
if st.session_state.vector_store:
|
| 69 |
+
stored_docs = {
|
| 70 |
+
meta.get('document_id')
|
| 71 |
+
for meta in st.session_state.vector_store.docstore.get_all_metadatas()
|
| 72 |
+
}
|
| 73 |
+
if set(document_ids) == stored_docs:
|
| 74 |
# Reuse existing vector store
|
| 75 |
+
st.session_state.qa_system = initialize_qa_system(st.session_state.vector_store)
|
| 76 |
+
st.session_state.chat_ready = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
+
# Initialize new vector store if needed
|
| 80 |
+
embeddings = get_embeddings_model()
|
| 81 |
+
text_splitter = RecursiveCharacterTextSplitter(
|
| 82 |
+
chunk_size=500,
|
| 83 |
+
chunk_overlap=50,
|
| 84 |
+
length_function=len,
|
| 85 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
|
| 87 |
+
chunks = []
|
| 88 |
+
for doc in documents:
|
| 89 |
+
doc_chunks = text_splitter.split_text(doc['content'])
|
| 90 |
+
chunks.extend([{
|
| 91 |
+
'content': chunk,
|
| 92 |
+
'metadata': {
|
| 93 |
+
'source': doc['name'],
|
| 94 |
+
'document_id': doc['id']
|
| 95 |
+
}
|
| 96 |
+
} for chunk in doc_chunks])
|
| 97 |
+
|
| 98 |
+
vector_store = FAISS.from_texts(
|
| 99 |
+
[chunk['content'] for chunk in chunks],
|
| 100 |
+
embeddings,
|
| 101 |
+
[chunk['metadata'] for chunk in chunks]
|
| 102 |
+
)
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
+
st.session_state.vector_store = vector_store
|
| 105 |
+
st.session_state.qa_system = initialize_qa_system(vector_store)
|
| 106 |
+
st.session_state.chat_ready = True
|
| 107 |
+
return True
|
| 108 |
+
|
| 109 |
+
except Exception as e:
|
| 110 |
+
st.error(f"Error initializing chat system: {e}")
|
| 111 |
+
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
def display_welcome_screen():
|
| 114 |
+
"""Display enhanced welcome screen with quick actions."""
|
| 115 |
+
st.title("๐ค Welcome to Document Analysis")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
col1, col2 = st.columns(2)
|
| 118 |
with col1:
|
| 119 |
+
# Quick Start Section
|
| 120 |
+
st.header("Quick Start")
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
+
# Upload new documents
|
| 123 |
+
uploaded_files = st.file_uploader(
|
| 124 |
+
"Upload Documents",
|
| 125 |
+
type=['pdf'],
|
| 126 |
+
accept_multiple_files=True,
|
| 127 |
+
help="Upload PDF documents to start analyzing"
|
| 128 |
+
)
|
| 129 |
|
| 130 |
+
if uploaded_files:
|
| 131 |
+
if handle_document_upload(uploaded_files):
|
| 132 |
+
initialize_chat_system()
|
| 133 |
+
st.rerun()
|
| 134 |
+
|
| 135 |
+
# Or select existing collection
|
| 136 |
+
st.markdown("### Or Select Existing Collection")
|
| 137 |
+
collections = get_collections(st.session_state.db_conn)
|
| 138 |
+
if collections:
|
| 139 |
+
for collection in collections:
|
| 140 |
+
if st.button(f"๐ {collection['name']} ({collection['doc_count']} documents)"):
|
| 141 |
+
st.session_state.selected_collection = collection
|
| 142 |
+
if initialize_chat_system(collection['id']):
|
| 143 |
st.rerun()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
with col2:
|
| 146 |
+
st.header("Recent Activity")
|
| 147 |
+
# Show recent documents or collections
|
| 148 |
+
documents = get_all_documents(st.session_state.db_conn)
|
| 149 |
+
if documents:
|
| 150 |
+
st.markdown("### Recent Documents")
|
| 151 |
+
for doc in documents[:5]: # Show last 5 documents
|
| 152 |
+
if st.button(f"๐ {doc['name']}", key=f"doc_{doc['id']}"):
|
| 153 |
+
if initialize_chat_system():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
st.rerun()
|
| 155 |
|
| 156 |
+
def display_chat_interface():
|
| 157 |
+
"""Display the main chat interface."""
|
| 158 |
+
st.header("๐ฌ Chat Interface")
|
| 159 |
+
|
| 160 |
+
# Display chat messages
|
| 161 |
+
for message in st.session_state.messages:
|
| 162 |
+
with st.chat_message("user" if isinstance(message, HumanMessage) else "assistant"):
|
| 163 |
+
st.markdown(message.content)
|
| 164 |
+
|
| 165 |
+
# Chat input
|
| 166 |
+
if prompt := st.chat_input("Ask a question about your documents..."):
|
| 167 |
+
st.session_state.messages.append(HumanMessage(content=prompt))
|
| 168 |
+
|
| 169 |
+
with st.spinner("Thinking..."):
|
| 170 |
+
response = st.session_state.qa_system.invoke({
|
| 171 |
+
"input": prompt,
|
| 172 |
+
"chat_history": st.session_state.messages
|
| 173 |
+
})
|
| 174 |
+
st.session_state.messages.append(AIMessage(content=response.content))
|
| 175 |
+
|
| 176 |
+
st.rerun()
|
| 177 |
|
| 178 |
def main():
|
| 179 |
+
"""Main application function with improved state management."""
|
|
|
|
| 180 |
st.set_page_config(
|
| 181 |
+
page_title="Document Analysis",
|
| 182 |
layout="wide",
|
|
|
|
| 183 |
initial_sidebar_state="expanded"
|
| 184 |
)
|
| 185 |
|
| 186 |
+
# Initialize session state and database
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
initialize_session_state()
|
| 188 |
|
| 189 |
+
# Database initialization
|
| 190 |
+
if 'db_conn' not in st.session_state:
|
| 191 |
+
db_path = Path('data/analysis.db')
|
| 192 |
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
| 193 |
+
st.session_state.db_conn = create_connection(str(db_path))
|
| 194 |
+
create_tables(st.session_state.db_conn)
|
| 195 |
+
verify_database_tables(st.session_state.db_conn)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
|
| 197 |
+
# Sidebar controls
|
| 198 |
with st.sidebar:
|
| 199 |
+
st.title("Navigation")
|
|
|
|
| 200 |
|
| 201 |
+
if st.button("๐ Home", use_container_width=True):
|
| 202 |
+
st.session_state.chat_ready = False
|
|
|
|
| 203 |
st.rerun()
|
| 204 |
|
| 205 |
+
if st.session_state.chat_ready:
|
| 206 |
+
if st.button("๐ New Chat", use_container_width=True):
|
| 207 |
+
st.session_state.messages = []
|
| 208 |
+
st.rerun()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
# Main content area
|
| 211 |
+
if not st.session_state.chat_ready:
|
|
|
|
|
|
|
| 212 |
display_welcome_screen()
|
| 213 |
+
else:
|
| 214 |
+
display_chat_interface()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
|
| 216 |
if __name__ == "__main__":
|
| 217 |
main()
|