Spaces:

Arthysp
/

summarize

Runtime error

Update app.py

0d15e0b verified over 2 years ago

2.27 kB

import os
import requests
import PyPDF2

from dotenv import load_dotenv
from langchain.text_splitter import RecursiveCharacterTextSplitter
from pinecone import Pinecone
from groq import Groq
from sentence_transformers import SentenceTransformer

em_model = SentenceTransformer("all-MiniLM-L6-v2")
def get_query_embdedding(embed):
    query_embedding = em_model.encode([embed]).tolist()   
    return  query_embedding
# Load the environment variables from the .env file
load_dotenv()

# Access the variables using os.getenv
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")


# Read the PDF file
pdf = PyPDF2.PdfReader("pdf_files/bhagat_singh_upsc_notes_60.pdf")
pdf_text = ""
for page in pdf.pages:
    pdf_text += page.extract_text()

# Split the text into chunks
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
texts = text_splitter.split_text(pdf_text)
# print(texts)
# Embedding the chunks
    

# embeddings = OllamaEmbeddings(model="nomic-embed-text")
# r1 = embeddings.embed_documents(texts)
r1 =  em_model.encode(texts)


# print (r1)
# Initialize Pinecone
pc = Pinecone(api_key = PINECONE_API_KEY)

# Create or connect to an existing index
index = pc.Index("sample2")

# Upsert the embeddings into the index
for i in range(len(texts)):
    index.upsert([((str(i), r1[i], {"text": texts[i]}))])

# Define the query
query = "Who is Bhagat singh?"

# Get the query embedding
question_embedding = get_query_embdedding(query)

# Query the Pinecone index
query_result = index.query(vector=question_embedding, top_k=5, include_metadata=True)

# Extract metadata from query result
docs = {x["metadata"]['text']: i for i, x in enumerate(query_result["matches"])}
# print (docs)
 
# Create a template for the summary

Template = f"Based on the following context: {docs} generate a precise summary related to the question: {query}"

# print(Template)
# Initialize Groq client
client = Groq(api_key = GROQ_API_KEY)

# Generate the summary
chat_completion = client.chat.completions.create(
    messages=[
        {
            "role": "user",
            "content": Template,
        }
    ],
    model="llama3-70b-8192",
)

# Print the summary
print(chat_completion.choices[0].message.content)


	import os
	import requests
	import PyPDF2

	from dotenv import load_dotenv
	from langchain.text_splitter import RecursiveCharacterTextSplitter
	from pinecone import Pinecone
	from groq import Groq
	from sentence_transformers import SentenceTransformer

	em_model = SentenceTransformer("all-MiniLM-L6-v2")
	def get_query_embdedding(embed):
	query_embedding = em_model.encode([embed]).tolist()
	return query_embedding
	# Load the environment variables from the .env file
	load_dotenv()

	# Access the variables using os.getenv
	GROQ_API_KEY = os.getenv("GROQ_API_KEY")
	PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")


	# Read the PDF file
	pdf = PyPDF2.PdfReader("pdf_files/bhagat_singh_upsc_notes_60.pdf")
	pdf_text = ""
	for page in pdf.pages:
	pdf_text += page.extract_text()

	# Split the text into chunks
	text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
	texts = text_splitter.split_text(pdf_text)
	# print(texts)
	# Embedding the chunks


	# embeddings = OllamaEmbeddings(model="nomic-embed-text")
	# r1 = embeddings.embed_documents(texts)
	r1 = em_model.encode(texts)


	# print (r1)
	# Initialize Pinecone
	pc = Pinecone(api_key = PINECONE_API_KEY)

	# Create or connect to an existing index
	index = pc.Index("sample2")

	# Upsert the embeddings into the index
	for i in range(len(texts)):
	index.upsert([((str(i), r1[i], {"text": texts[i]}))])

	# Define the query
	query = "Who is Bhagat singh?"

	# Get the query embedding
	question_embedding = get_query_embdedding(query)

	# Query the Pinecone index
	query_result = index.query(vector=question_embedding, top_k=5, include_metadata=True)

	# Extract metadata from query result
	docs = {x["metadata"]['text']: i for i, x in enumerate(query_result["matches"])}
	# print (docs)

	# Create a template for the summary

	Template = f"Based on the following context: {docs} generate a precise summary related to the question: {query}"

	# print(Template)
	# Initialize Groq client
	client = Groq(api_key = GROQ_API_KEY)

	# Generate the summary
	chat_completion = client.chat.completions.create(
	messages=[
	{
	"role": "user",
	"content": Template,
	}
	],
	model="llama3-70b-8192",
	)

	# Print the summary
	print(chat_completion.choices[0].message.content)