File size: 3,780 Bytes
83ba83f
a75e954
 
83ba83f
a75e954
 
83ba83f
 
a75e954
 
83ba83f
 
a75e954
83ba83f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import streamlit as st
import fitz
import pandas as pd
import numpy as np
import re
import torch
from tqdm.auto import tqdm
from sentence_transformers import SentenceTransformer, util
import textwrap

# Set device
device = "cuda" if torch.cuda.is_available() else "cpu"

# Load PDF and process
@st.cache_data(show_spinner=True)
def process_pdf(pdf_path):
    def text_formatter(text):
        return text.replace("\n", " ").strip()

    doc = fitz.open(pdf_path)
    pages_and_texts = []
    for page_number, page in enumerate(doc):
        text = text_formatter(page.get_text())
        pages_and_texts.append({
            "page_number": page_number - 15,
            "text": text,
            "page_char_count": len(text),
            "page_word_count": len(text.split(" ")),
            "page_sentence_count_raw": len(text.split(". ")),
            "page_token_count": len(text)/4
        })
    return pages_and_texts

@st.cache_data(show_spinner=True)
def chunk_sentences(pages_and_texts, chunk_size=10):
    import spacy
    nlp = spacy.blank("en")
    nlp.add_pipe("sentencizer")

    def split_list(lst, size):
        return [lst[i:i+size] for i in range(0, len(lst), size)]

    for item in pages_and_texts:
        item["sentences"] = [str(s) for s in nlp(item["text"]).sents]
        item["sentence_chunks"] = split_list(item["sentences"], chunk_size)

    pages_and_chunks = []
    for item in pages_and_texts:
        for chunk in item["sentence_chunks"]:
            joined = "".join(chunk).replace("  ", " ").strip()
            joined = re.sub(r'\.([A-Z])', r'. \1', joined)
            pages_and_chunks.append({
                "page_number": item["page_number"],
                "sentence_chunk": joined,
                "chunk_char_count": len(joined),
                "chunk_word_count": len(joined.split(" ")),
                "chunk_token_count": len(joined)/4
            })
    return pages_and_chunks

@st.cache_data(show_spinner=True)
def embed_chunks(pages_and_chunks):
    model = SentenceTransformer("all-mpnet-base-v2", device=device)
    filtered = [c for c in pages_and_chunks if c["chunk_token_count"] > 30]
    texts = [item["sentence_chunk"] for item in filtered]
    embeddings = model.encode(texts, convert_to_tensor=True)
    for i, emb in enumerate(embeddings):
        filtered[i]["embedding"] = emb.cpu().numpy()
    return filtered, embeddings, model

@st.cache_data(show_spinner=False)
def semantic_search(query, embeddings, model, chunks, k=5):
    query_emb = model.encode(query, convert_to_tensor=True).to(device)
    scores = util.dot_score(query_emb, embeddings)[0]
    top_k = torch.topk(scores, k)
    results = []
    for score, idx in zip(top_k[0], top_k[1]):
        results.append({
            "score": float(score),
            "text": chunks[idx]["sentence_chunk"],
            "page": chunks[idx]["page_number"]
        })
    return results

st.title("Semantic Search App for Ian Goodfellow's Deep Learning Book")

uploaded_file = st.file_uploader("Upload the PDF", type="pdf")

if uploaded_file:
    with st.spinner("Processing PDF..."):
        with open("uploaded.pdf", "wb") as f:
            f.write(uploaded_file.read())
        texts = process_pdf("uploaded.pdf")
        chunks = chunk_sentences(texts)
        chunk_data, chunk_embeddings, emb_model = embed_chunks(chunks)

    query = st.text_input("Enter your query:")
    if query:
        with st.spinner("Searching..."):
            results = semantic_search(query, chunk_embeddings, emb_model, chunk_data)
        st.subheader("Top Results")
        for res in results:
            st.markdown(f"**Score:** {res['score']:.4f}")
            st.markdown(f"**Page:** {res['page']}")
            st.markdown(f"> {res['text']}")
            st.markdown("---")