File size: 2,266 Bytes
9e2b4e0 c929282 9e2b4e0 0d15e0b e0eb35e 9e2b4e0 c929282 9e2b4e0 4893316 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import os
import requests
import PyPDF2
from dotenv import load_dotenv
from langchain.text_splitter import RecursiveCharacterTextSplitter
from pinecone import Pinecone
from groq import Groq
from sentence_transformers import SentenceTransformer
em_model = SentenceTransformer("all-MiniLM-L6-v2")
def get_query_embdedding(embed):
query_embedding = em_model.encode([embed]).tolist()
return query_embedding
# Load the environment variables from the .env file
load_dotenv()
# Access the variables using os.getenv
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
# Read the PDF file
pdf = PyPDF2.PdfReader("pdf_files/bhagat_singh_upsc_notes_60.pdf")
pdf_text = ""
for page in pdf.pages:
pdf_text += page.extract_text()
# Split the text into chunks
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
texts = text_splitter.split_text(pdf_text)
# print(texts)
# Embedding the chunks
# embeddings = OllamaEmbeddings(model="nomic-embed-text")
# r1 = embeddings.embed_documents(texts)
r1 = em_model.encode(texts)
# print (r1)
# Initialize Pinecone
pc = Pinecone(api_key = PINECONE_API_KEY)
# Create or connect to an existing index
index = pc.Index("sample2")
# Upsert the embeddings into the index
for i in range(len(texts)):
index.upsert([((str(i), r1[i], {"text": texts[i]}))])
# Define the query
query = "Who is Bhagat singh?"
# Get the query embedding
question_embedding = get_query_embdedding(query)
# Query the Pinecone index
query_result = index.query(vector=question_embedding, top_k=5, include_metadata=True)
# Extract metadata from query result
docs = {x["metadata"]['text']: i for i, x in enumerate(query_result["matches"])}
# print (docs)
# Create a template for the summary
Template = f"Based on the following context: {docs} generate a precise summary related to the question: {query}"
# print(Template)
# Initialize Groq client
client = Groq(api_key = GROQ_API_KEY)
# Generate the summary
chat_completion = client.chat.completions.create(
messages=[
{
"role": "user",
"content": Template,
}
],
model="llama3-70b-8192",
)
# Print the summary
print(chat_completion.choices[0].message.content) |