Spaces:
Build error
Build error
Download app.py from FIlcor/Chat_to_doc: direct link, hf CLI and curl.
- Browser
- Download file 6.45 kB
-
https://huggingface.co/spaces/FIlcor/Chat_to_doc/resolve/main/app.py
- Command line
-
hf download hf://spaces/FIlcor/Chat_to_doc/app.py
-
curl -L -o app.py https://huggingface.co/spaces/FIlcor/Chat_to_doc/resolve/main/app.py
6.45 kB
| import streamlit as st | |
| import openai | |
| import PyPDF2 | |
| import io | |
| import numpy as np | |
| import faiss | |
| from io import BytesIO | |
| from wand.image import Image | |
| from wand.color import Color | |
| import pytesseract | |
| import tiktoken | |
| import re | |
| from pdf2image import convert_from_path | |
| import openai.api_resources.completion | |
| from streamlit_chat import message | |
| # Function to extract text from PDF | |
| def extract_text_from_pdf(pdf_file): | |
| # Use BytesIO to read the file content directly | |
| with BytesIO(pdf_file.getbuffer()) as file: | |
| pdf_reader = PyPDF2.PdfReader(file) | |
| text = '' | |
| for page in pdf_reader.pages: | |
| text += page.extract_text() | |
| if len(text) == 0: | |
| # Convert BytesIO back to a temporary file for image extraction | |
| with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_pdf: | |
| temp_pdf.write(file.getvalue()) | |
| temp_pdf.flush() | |
| text = extract_text_from_pdf_image(temp_pdf.name) | |
| text = clean_text(text) | |
| else: | |
| text = clean_text(text) | |
| return text | |
| def extract_text_from_pdf_image(pdf_file): | |
| try: | |
| # Convert PDF to images | |
| images = convert_from_path(pdf_file, dpi=300) | |
| text = '' | |
| for image in images: | |
| # Extract text using OCR | |
| text += pytesseract.image_to_string(image, lang='ita', config='--psm 1') + '\n' | |
| return text | |
| except Exception as e: # Replace FileNotFoundError with Exception | |
| print("Error:", e) | |
| def clean_text(text): | |
| # rimuove spazi multipli | |
| text = re.sub(r'\s+', ' ', text) | |
| # rimuove caratteri di nuova linea e tabulazioni | |
| text = re.sub(r'[\n\t]', '', text) | |
| # rimuove caratteri non ASCII | |
| text = re.sub(r'[^\x00-\x7f]', r'', text) | |
| # rimuove spazi all'inizio e alla fine del testo | |
| text = text.strip() | |
| return text | |
| # Function to chunk text | |
| def chunk_text(text, max_length=2000): | |
| words = text.split() | |
| chunks = [] | |
| current_chunk = [] | |
| for word in words: | |
| current_chunk.append(word) | |
| if len(" ".join(current_chunk)) > max_length: | |
| current_chunk.pop() | |
| chunks.append(" ".join(current_chunk)) | |
| current_chunk = [word] | |
| if current_chunk: | |
| chunks.append(" ".join(current_chunk)) | |
| return chunks | |
| # Function to get embeddings | |
| def get_embedding(text, model="text-embedding-ada-002"): | |
| text = text.replace("\n", " ") | |
| return openai.Embedding.create(input=[text], model=model)['data'][0]['embedding'] | |
| # Function to interact with chatbot | |
| def interact_with_chatbot(context, message, max_response_length=200): | |
| prompt = f"{message}\n\n" | |
| for idx, chunk in enumerate(context): | |
| prompt += f"Context-{idx + 1}: {chunk}\n\n" | |
| response = openai.ChatCompletion.create( | |
| model="gpt-3.5-turbo", | |
| messages=[{"role": "system", "content": "Sei un bot che risponde alle domande che l'utente ti fa in base al contesto che ti viene dato, se la risposta non è disponibile nel contesto allora prova ad usare la tu abase di conoscenza, in questo caso però fai presente che l'informazione non proviene dal contesto dato."}, {"role": "user", "content": prompt}], | |
| max_tokens=max_response_length, | |
| n=1, | |
| stop=None, | |
| temperature=0.5, | |
| top_p=1, | |
| frequency_penalty=0, | |
| presence_penalty=0, | |
| ) | |
| return response['choices'][0]['message']['content'].strip() | |
| conversation_history = [] | |
| # Main app | |
| def main(): | |
| st.title("Chat with PDF") | |
| # Set OpenAI API key | |
| st.sidebar.title("API Key") | |
| api_key = st.sidebar.text_input("Enter your OpenAI API key", type="password") | |
| openai.api_key = api_key | |
| # Upload PDFs | |
| st.sidebar.title("Upload PDFs") | |
| pdf_files = st.sidebar.file_uploader("Select one or more PDFs", type="pdf", accept_multiple_files=True) | |
| #Settings | |
| st.sidebar.title("Impostazioni") | |
| max_response_length = st.sidebar.slider("Lunghezza massima della risposta", min_value=100, max_value=1000, value=200) | |
| # Extract and store text from PDFs | |
| pdf_texts = {} | |
| for pdf_file in pdf_files: | |
| pdf_name = pdf_file.name | |
| pdf_texts[pdf_name] = extract_text_from_pdf(pdf_file) | |
| # Embed and store the text | |
| vectorial_db = {} # To store embeddings along with the PDF names and text chunks | |
| for pdf_name, text in pdf_texts.items(): | |
| chunks = chunk_text(text) | |
| embeddings = [get_embedding(chunk) for chunk in chunks] | |
| vectorial_db[pdf_name] = { | |
| 'chunks': chunks, | |
| 'embeddings': embeddings, | |
| } | |
| # Index embeddings in a vectorial database | |
| if vectorial_db: # Add this check to ensure vectorial_db is not empty | |
| embedding_size = len(vectorial_db[list(vectorial_db.keys())[0]]['embeddings'][0]) | |
| index = faiss.IndexFlatL2(embedding_size) | |
| for pdf_name, data in vectorial_db.items(): | |
| for embedding in data['embeddings']: | |
| index.add(np.array([embedding])) | |
| # Chatbot interaction | |
| # Initialize the conversation history | |
| if 'generated' not in st.session_state: | |
| st.session_state['generated'] = [] | |
| if 'past' not in st.session_state: | |
| st.session_state['past'] = [] | |
| user_input = st.text_input("Type your message here") | |
| if user_input: | |
| context = [] # Initialize an empty context list | |
| if vectorial_db: # Add this check to ensure vectorial_db is not empty | |
| # Query vectorial database | |
| user_embedding = get_embedding(user_input) | |
| D, I = index.search(np.array([user_embedding]), 5) | |
| # Set context | |
| context = [vectorial_db[pdf_name]['chunks'][embedding_idx] for pdf_name, embedding_idx in zip(vectorial_db.keys(), I[0])] | |
| response = interact_with_chatbot(context, user_input, max_response_length) | |
| st.session_state.past.append(user_input) | |
| st.session_state.generated.append(response) | |
| if st.session_state['generated']: | |
| for i in range(len(st.session_state['generated']) - 1, -1, -1): | |
| message(st.session_state["generated"][i], key=str(i)) | |
| message(st.session_state['past'][i], is_user=True, key=str(i) + '_user') | |
| # Manage context | |
| st.sidebar.title("Gestione Informazioni") | |
| clear_context = st.sidebar.button("Cancella Informazioni") | |
| if clear_context: | |
| context = [] | |
| if __name__ == "__main__": | |
| main() |