Spaces:
Runtime error
Runtime error
File size: 18,488 Bytes
4588f44 a5fd983 4588f44 c9f26ef 7353cb2 4588f44 d79a838 a3b941b a5fd983 4588f44 249a6b8 c5aa90d 4588f44 337de48 349a8c1 249a6b8 349a8c1 4588f44 249a6b8 c9f26ef c5aa90d d79a838 37d2255 d79a838 c5aa90d d79a838 c5aa90d d79a838 c5aa90d 2ddcc25 37d2255 2ddcc25 4588f44 fc6e95c 2d9a907 c5aa90d 2d9a907 fc6e95c 2d9a907 c5aa90d 2ddcc25 e508e34 c5aa90d e508e34 c5aa90d 2ddcc25 e508e34 fc6e95c e508e34 2d9a907 e508e34 c5aa90d e508e34 2d9a907 4588f44 fc6e95c 863e2b0 1b6aa28 c5aa90d 863e2b0 1b6aa28 249a6b8 863e2b0 249a6b8 863e2b0 249a6b8 863e2b0 249a6b8 863e2b0 4588f44 1376706 a3b941b 1376706 bb8c38b 249a6b8 7683b60 349a8c1 37d2255 4588f44 4260bc7 4588f44 c5aa90d 7683b60 7864140 4588f44 4260bc7 4588f44 249a6b8 534a4c9 249a6b8 4260bc7 4588f44 4260bc7 0bff2bb c5f80e9 a197692 4588f44 fc6e95c 8754903 e508e34 bb8c38b e635a75 bb8c38b fc6e95c 249a6b8 863e2b0 4588f44 249a6b8 7864140 fd2d13f 4588f44 fd2d13f 863e2b0 249a6b8 c2b949f 249a6b8 c2b949f 05862aa a197692 05862aa aaffd11 a1f010d aaffd11 bb8c38b a1f010d aaffd11 927d5fd aaffd11 927d5fd 8754903 a1f010d aaffd11 a197692 d21c409 5407bae d21c409 a197692 249a6b8 aaffd11 acfede2 a197692 8754903 249a6b8 a197692 927d5fd a197692 acfede2 bb8c38b acfede2 0a66fc2 acfede2 bb8c38b a197692 acfede2 c066c05 acfede2 0a66fc2 acfede2 c5ca649 c2b949f 8754903 bb8c38b fc6e95c bb8c38b 8754903 bb8c38b fc6e95c fd2d13f 4588f44 8754903 a197692 fd2d13f 249a6b8 a197692 fd2d13f 249a6b8 a197692 c2b949f bb8c38b 8754903 e635a75 8754903 bb8c38b fc6e95c bb8c38b c316d24 8754903 c316d24 bb8c38b e635a75 8754903 249a6b8 863e2b0 4588f44 01422c5 da6d132 1bc2878 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 | import os
import json
import uuid
from datetime import datetime
from typing import List
import gradio as gr
from dotenv import load_dotenv
import fitz # Import PyMuPDF as fitz
print(f"PyMuPDF version: {fitz.__version__}") # Debug2
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain.prompts import PromptTemplate
from qdrant_client import QdrantClient
from qdrant_client.http import models
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import re
import time
load_dotenv()
class PdfProcessor:
def __init__(self):
self.embeddings = OpenAIEmbeddings(model="text-embedding-3-small", openai_api_key=os.getenv("OPENAI_API_KEY"))
self.qdrant_client = QdrantClient(
url=os.getenv("QDRANT_URL"),
api_key=os.getenv("QDRANT_API_KEY"),
timeout=60
)
self.collection_name = "document_embeddings"
self._ensure_collection_exists()
def _ensure_collection_exists(self):
"""Ensure the Qdrant collection exists, creating it if necessary."""
if not self.qdrant_client.collection_exists(collection_name=self.collection_name):
self.qdrant_client.create_collection(
collection_name=self.collection_name,
vectors_config=models.VectorParams(size=1536, distance=models.Distance.COSINE)
)
print(f"Created collection: {self.collection_name}") # Debug
def _extract_text(self, file_path):
"""Extract text from a PDF using PyMuPDF."""
text = ""
with fitz.open(file_path) as pdf:
for page in pdf:
text += page.get_text("text") or "" # Extract text from each page
return text
def _chunk_text(self, text, max_chunk_size=500, min_chunk_size=100, similarity_threshold=0.5):
"""Performs semantic chunking using embeddings and cosine similarity."""
sentences = re.split(r'(?<=[.!?])\s+', text)
sentence_embeddings = self.embeddings.embed_documents(sentences)
chunks, chunk, chunk_embedding = [], [], []
chunk_token_length = 0
for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
if not chunk:
chunk.append(sentence)
chunk_embedding.append(embedding)
chunk_token_length = len(sentence)
continue
similarity = cosine_similarity([embedding], [chunk_embedding[-1]])[0][0]
if chunk_token_length + len(sentence) > max_chunk_size:
if chunk_token_length >= min_chunk_size:
chunks.append(" ".join(chunk))
chunk = [sentence]
chunk_embedding = [embedding]
chunk_token_length = len(sentence)
elif similarity > similarity_threshold:
chunk.append(sentence)
chunk_embedding.append(embedding)
chunk_token_length += len(sentence)
else:
if chunk_token_length >= min_chunk_size:
chunks.append(" ".join(chunk))
chunk = [sentence]
chunk_embedding = [embedding]
chunk_token_length = len(sentence)
if chunk and (chunk_token_length >= min_chunk_size or len(chunks) == 0):
chunks.append(" ".join(chunk))
unique_chunks = list(dict.fromkeys(chunks))
return unique_chunks
def _generate_summary(self, text: str) -> str:
sentences = [s.strip() for s in text.split('.') if s.strip()]
summary = '. '.join(sentences[:2]) if len(sentences) >= 2 else text[:100]
return summary if summary.endswith('.') else summary + '.'
def _generate_full_summary(self, text: str) -> str:
"""Generate a 5-point summary for the entire document."""
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
prompt = PromptTemplate(
input_variables=["text"],
template="""Generate a concise 5-point summary of the following text in bullet point format:
{text}
Summary:"""
)
chain = prompt | llm
summary = chain.invoke({"text": text[:10000]}).content.strip() # Limit text to avoid token limits
return summary
def process_and_store_pdf(self, file_obj):
self._ensure_collection_exists()
text = self._extract_text(file_obj.name)
filename = os.path.basename(file_obj.name)
chunks = self._chunk_text(text)
full_summary = self._generate_full_summary(text)
points = []
batch_size = 100
seen_chunks = set()
for i in range(0, len(chunks), batch_size):
batch_chunks = chunks[i:i + batch_size]
embeddings = self.embeddings.embed_documents(batch_chunks)
for j, (chunk, embedding) in enumerate(zip(batch_chunks, embeddings)):
if chunk in seen_chunks:
continue
seen_chunks.add(chunk)
point_id = str(uuid.uuid4())
summary = self._generate_summary(chunk)
points.append(
models.PointStruct(
id=point_id,
vector=embedding,
payload={
"text": chunk,
"source_file": filename,
"chunk_id": i + j,
"summary": summary,
"original_file_name": filename,
"conversion_date": datetime.now().isoformat(),
"full_summary": full_summary
}
)
)
if points:
self.qdrant_client.upsert(collection_name=self.collection_name, points=points)
points = []
return filename
def get_document_summary(self, filename: str) -> str:
"""Retrieve the full summary for a specific document."""
scroll_response = self.qdrant_client.scroll(
collection_name=self.collection_name,
scroll_filter=models.Filter(
must=[
models.FieldCondition(
key="original_file_name",
match=models.MatchValue(value=filename)
)
]
),
limit=1,
with_payload=True
)
points, _ = scroll_response
if points and "full_summary" in points[0].payload:
return points[0].payload["full_summary"]
return "No summary available."
def get_processed_files(self):
processed_files=()
self._ensure_collection_exists()
processed_files = set()
scroll_response = self.qdrant_client.scroll(
collection_name=self.collection_name,
limit=100,
with_payload=True
)
points, next_offset = scroll_response
while points:
for point in points:
original_file_name = point.payload.get("original_file_name")
if original_file_name:
processed_files.add(original_file_name)
if next_offset is None:
break
scroll_response = self.qdrant_client.scroll(
collection_name=self.collection_name,
limit=100,
offset=next_offset,
with_payload=True
)
points, next_offset = scroll_response
return processed_files
def delete_file_points(self, filename: str):
self._ensure_collection_exists()
filter_condition = models.Filter(
must=[
models.FieldCondition(
key="original_file_name",
match=models.MatchValue(value=filename)
)
]
)
self.qdrant_client.delete(
collection_name=self.collection_name,
points_selector=models.FilterSelector(filter=filter_condition)
)
time.sleep(0.5) # Small delay to ensure Qdrant processes the deletion
return f"Deleted points for file: {filename}"
def query_vector_db(query: str) -> dict:
qdrant_client = QdrantClient(
url=os.getenv("QDRANT_URL"),
api_key=os.getenv("QDRANT_API_KEY")
)
if not qdrant_client.collection_exists(collection_name="document_embeddings"):
qdrant_client.create_collection(
collection_name="document_embeddings",
vectors_config=models.VectorParams(size=1536, distance=models.Distance.COSINE)
)
print("Created collection 'document_embeddings' in query_vector_db")
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
query_embedding = embeddings.embed_query(query)
query_response = qdrant_client.query_points(
collection_name="document_embeddings",
query=query_embedding,
limit=5,
with_payload=True
)
search_results = query_response.points
if not search_results:
return {"context": "No relevant information found.", "source": "None", "chunks": []}
context = "\n\n".join(
[result.payload.get("text", "No text available") for result in search_results]
)
chunks = [result.payload.get("text", "No text available") for result in search_results]
source = search_results[0].payload.get("original_file_name", "Unknown")
return {"context": context, "source": source, "chunks": chunks}
def classify_relevant_file(question, processor):
processed_files = processor.get_processed_files()
return list(processed_files)[0] if processed_files else None
def query_documents(question, processor):
relevant_file = classify_relevant_file(question, processor)
tool_output = query_vector_db(query=question)
answer_prompt = PromptTemplate(
input_variables=["question", "context"],
template="""Based solely on the provided context, provide a concise, accurate answer to the question. Do not use external knowledge.
Question: {question}
Context: {context}
Answer: """
)
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
chain = answer_prompt | llm
answer = chain.invoke({"question": question, "context": tool_output["context"]}).content.strip()
if "context does not" in answer:
final_answer = answer
else:
final_answer = f"{answer} [Source: {tool_output['source']}]"
return {"answer": final_answer, "chunks": tool_output["chunks"]}
def delete_file(selected_files):
if selected_files and len(selected_files) > 0:
selected_file = selected_files[0]
processor.delete_file_points(selected_file)
updated_choices = sorted(processor.get_processed_files())
return f"Deleted: {selected_file}", updated_choices
return "No file selected", sorted(processor.get_processed_files())
def summarize_document(selected_files, history):
if not selected_files or len(selected_files) == 0:
return history, "No file selected for summary"
selected_file = selected_files[0]
summary = processor.get_document_summary(selected_file)
summary_message = f"""5 Point Summary of {selected_file}
{summary}
If you need specific information, type your question in the box below and click Send."""
new_history = history + [
{"role": "assistant", "content": summary_message}
]
return new_history, f"Summarized: {selected_file}"
processor = PdfProcessor()
def chat_function(message, history):
response = query_documents(message, processor)
answer = response["answer"]
chunks = "\n\n".join([f"Chunk {i+1}: {chunk}" for i, chunk in enumerate(response["chunks"])])
new_history = history + [
{"role": "user", "content": message},
{"role": "assistant", "content": answer}
]
return new_history, "", chunks
def upload_file(file):
if file is None:
return "Please upload a PDF file"
filename = processor.process_and_store_pdf(file)
return f"Processed: {filename}"
modern_theme = gr.themes.Monochrome(
primary_hue="indigo",
secondary_hue="gray",
neutral_hue="slate",
radius_size="lg",
text_size="md",
)
custom_css = """
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap');
* { font-family: 'Inter', sans-serif !important; }
.container { max-width: 1200px; margin: 0 auto; }
.header { text-align: center; padding: 1rem 0; }
.chatbot .message { border-radius: 10px; padding: 10px; margin: 5px 0; }
.user-message { background-color: #E94560 !important; color: white; }
.assistant-message { background-color: #0F3460 !important; color: white; }
.message-input { background-color: #0F3460; color: white; border: 1px solid #E94560; border-radius: 8px; }
.message-input textarea { height: 60px !important; }
.send-btn { background-color: #E94560; color: white; border: none; height: 60px !important; display: flex; align-items: center; justify-content: center; font-size: 18px !important; font-weight: 500; }
.send-btn:hover { background-color: #FF6B6B; }
.file-upload { background-color: #16213E; border: 1px dashed #E94560; border-radius: 8px; height: 200px; }
.status-text { color: #E94560; font-size: 14px; text-align: center; }
.info-box { background-color: #16213E; color: white; border: 1px solid #0F3460; border-radius: 8px; }
.gr-button { transition: all 0.3s ease; }
.gr-accordion { background-color: #16213E; border: 1px solid #0F3460; }
.square-column { min-width: 300px; max-width: 400px; }
.checkbox-group { max-height: 150px; overflow-y: auto; }
.gr-accordion > div > button { font-size: 18px !important; font-weight: 500; }
.header-text { color: white; font-size: 16px; margin-top: -10px; }
.warning-text { color: #E94560; font-size: 16px; margin-top: -10px; }
"""
with gr.Blocks(theme=modern_theme, css=custom_css) as demo:
gr.Markdown(
"""
# Zaplytic
<p style='color: #E94560; font-size: 16px; margin-top: -10px;'>Upload your document to store it in a vector database and chat with it to get instant answers.</p>
<p style='color: red; font-size: 16px; margin-top: -10px;'><b style='color: red;'>Do not upload personal or private data</b></p>
""",
elem_classes="header"
)
with gr.Row(elem_classes="container"):
with gr.Column(scale=7):
chatbot = gr.Chatbot(
label=None,
type="messages",
height=500,
elem_classes="info-box"
)
with gr.Row():
msg = gr.Textbox(
placeholder="Ask anything...",
show_label=False,
container=False,
elem_classes="message-input",
lines=3,
scale=8
)
send_btn = gr.Button(
"Send",
size="sm",
elem_classes="send-btn",
scale=2
)
with gr.Column(scale=3):
with gr.Accordion("File Upload", open=False):
file_upload = gr.File(
label="Drop PDF here",
file_types=[".pdf"],
elem_classes="file-upload",
height=200
)
upload_btn = gr.Button(
"Upload PDF",
size="sm"
)
upload_status = gr.Textbox(
value="Upload a PDF to begin",
show_label=False,
interactive=False,
elem_classes="status-text"
)
with gr.Accordion("Manage Documents", open=False):
files_selection = gr.CheckboxGroup(
choices=sorted(processor.get_processed_files()),
label="Document Selection (Select one)",
interactive=True,
elem_classes="info-box checkbox-group"
)
with gr.Row():
summarize_btn = gr.Button(
"Summarize Document",
size="sm",
scale=1
)
delete_btn = gr.Button(
"Delete Document",
size="sm",
scale=1
)
chunks_display = gr.Textbox(
value="Query results will appear here",
label="Retrieved Chunks",
interactive=False,
lines=5,
max_lines=5,
elem_classes="info-box"
)
def update_selection(selected_files):
if len(selected_files) > 1:
return [selected_files[-1]]
return selected_files
send_btn.click(
chat_function,
inputs=[msg, chatbot],
outputs=[chatbot, msg, chunks_display]
)
msg.submit(
chat_function,
inputs=[msg, chatbot],
outputs=[chatbot, msg, chunks_display]
)
upload_btn.click(
upload_file,
inputs=[file_upload],
outputs=[upload_status]
).then(
lambda: gr.update(choices=sorted(processor.get_processed_files())),
outputs=[files_selection]
).then(
lambda: [], # Clear selection after upload
outputs=[files_selection]
)
files_selection.change(
update_selection,
inputs=[files_selection],
outputs=[files_selection]
)
summarize_btn.click(
summarize_document,
inputs=[files_selection, chatbot],
outputs=[chatbot, upload_status]
)
delete_btn.click(
delete_file,
inputs=[files_selection],
outputs=[upload_status, files_selection]
).then(
lambda: [], # Clear selection after deletion
outputs=[files_selection]
)
if __name__ == "__main__":
print(f"===== Application Startup at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} =====")
demo.launch(server_name="0.0.0.0", server_port=7860, show_api=False)
|