vmalapaka commited on
Commit
6636b26
·
verified ·
1 Parent(s): d55c9b7

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +19 -0
utils.py CHANGED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
2
+ from langchain_community.vectorstores import Pinecone
3
+ from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
4
+ from pinecone import Pinecone as PineconeClient
5
+ import asyncio
6
+ from langchain.document_loaders.sitemap import SitemapLoader
7
+
8
+
9
+ def get_website_data(sitemap_url):
10
+
11
+ loop = asyncio.new_event_loop()
12
+ asyncio.set_event_loop(loop)
13
+ loader = SitemapLoader(
14
+ sitemap_url
15
+ )
16
+
17
+ docs = loader.load()
18
+
19
+ return docs