import os import requests import PyPDF2 from dotenv import load_dotenv from langchain.text_splitter import RecursiveCharacterTextSplitter from pinecone import Pinecone from groq import Groq from sentence_transformers import SentenceTransformer em_model = SentenceTransformer("all-MiniLM-L6-v2") def get_query_embdedding(embed): query_embedding = em_model.encode([embed]).tolist() return query_embedding # Load the environment variables from the .env file load_dotenv() # Access the variables using os.getenv GROQ_API_KEY = os.getenv("GROQ_API_KEY") PINECONE_API_KEY = os.getenv("PINECONE_API_KEY") # Read the PDF file pdf = PyPDF2.PdfReader("pdf_files/bhagat_singh_upsc_notes_60.pdf") pdf_text = "" for page in pdf.pages: pdf_text += page.extract_text() # Split the text into chunks text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) texts = text_splitter.split_text(pdf_text) # print(texts) # Embedding the chunks # embeddings = OllamaEmbeddings(model="nomic-embed-text") # r1 = embeddings.embed_documents(texts) r1 = em_model.encode(texts) # print (r1) # Initialize Pinecone pc = Pinecone(api_key = PINECONE_API_KEY) # Create or connect to an existing index index = pc.Index("sample2") # Upsert the embeddings into the index for i in range(len(texts)): index.upsert([((str(i), r1[i], {"text": texts[i]}))]) # Define the query query = "Who is Bhagat singh?" # Get the query embedding question_embedding = get_query_embdedding(query) # Query the Pinecone index query_result = index.query(vector=question_embedding, top_k=5, include_metadata=True) # Extract metadata from query result docs = {x["metadata"]['text']: i for i, x in enumerate(query_result["matches"])} # print (docs) # Create a template for the summary Template = f"Based on the following context: {docs} generate a precise summary related to the question: {query}" # print(Template) # Initialize Groq client client = Groq(api_key = GROQ_API_KEY) # Generate the summary chat_completion = client.chat.completions.create( messages=[ { "role": "user", "content": Template, } ], model="llama3-70b-8192", ) # Print the summary print(chat_completion.choices[0].message.content)