Spaces:
Sleeping
Sleeping
File size: 2,840 Bytes
ac5b656 d2fbc5e ac5b656 d2fbc5e b3582ec ac5b656 d2fbc5e a8e46d8 ac5b656 d2fbc5e ac5b656 d2fbc5e ac5b656 d2fbc5e ac5b656 d2fbc5e ac5b656 d2fbc5e ac5b656 d2fbc5e b3582ec a8e46d8 d2fbc5e a8e46d8 d2fbc5e a8e46d8 d2fbc5e a8e46d8 d2fbc5e a8e46d8 d2fbc5e b3582ec ac5b656 d2fbc5e ac5b656 d2fbc5e ac5b656 a8e46d8 d2fbc5e a8e46d8 d2fbc5e a8e46d8 ac5b656 d2fbc5e a8e46d8 d2fbc5e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | import urllib.request
import fitz # PyMuPDF
import re
import numpy as np
import tensorflow_hub as hub
from sklearn.neighbors import NearestNeighbors
import os
import gradio as gr
def download_pdf(url, output_path):
try:
urllib.request.urlretrieve(url, output_path)
return True
except Exception as e:
print(f"Error downloading PDF: {e}")
return False
def preprocess(text):
text = text.replace('\n', ' ')
text = re.sub('\s+', ' ', text).strip()
return text
def pdf_to_text(path, start_page=1, end_page=None):
try:
doc = fitz.open(path)
total_pages = doc.page_count
if end_page is None:
end_page = total_pages
text_list = []
for i in range(start_page - 1, end_page):
page = doc.load_page(i)
text = page.get_text("text")
text = preprocess(text)
text_list.append(text)
doc.close()
return text_list
except Exception as e:
print(f"Error in PDF to text conversion: {e}")
return None
def text_to_chunks(texts, word_length=150, start_page=1):
text_tokens = [t.split(' ') for t in texts]
chunks = []
for idx, words in enumerate(text_tokens):
for i in range(0, len(words), word_length):
chunk = words[i:i + word_length]
chunk = ' '.join(chunk).strip()
chunk = f'[Page no. {idx + start_page}] ' + chunk
chunks.append(chunk)
return chunks
class SemanticSearch:
def __init__(self):
self.use = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
self.fitted = False
def fit(self, data):
self.data = data
self.embeddings = self.use(data)
self.nn = NearestNeighbors(n_neighbors=5)
self.nn.fit(self.embeddings)
self.fitted = True
def __call__(self, text):
if not self.fitted:
return "Model not fitted yet."
query_embedding = self.use([text])
neighbors = self.nn.kneighbors(query_embedding, return_distance=False)[0]
return [self.data[i] for i in neighbors]
recommender = SemanticSearch()
def gui(url, question):
if url.strip():
if not download_pdf(url, "temp.pdf"):
return "Failed to download PDF."
texts = pdf_to_text("temp.pdf")
if texts is None:
return "Failed to extract text from PDF."
chunks = text_to_chunks(texts)
recommender.fit(chunks)
else:
return "Please provide a valid URL."
if question.strip():
results = recommender(question)
return results
else:
return "Please enter a question."
iface = gr.Interface(
fn=gui,
inputs=["text", "text"],
outputs="text"
)
iface.launch()
|