File size: 2,531 Bytes
8d29339 5c312ac 7ff12f8 e24fd2c 7ff12f8 1f932d5 57232db 8d29339 3d5d389 7ff12f8 8d29339 2a6d4e6 7ff12f8 3d5d389 f6fff89 3d5d389 f26622c 7ff12f8 1aa1cba 7ff12f8 acd1740 7ff12f8 0395bcc 7ff12f8 acd1740 7ff12f8 cb638c9 acd1740 8322ba8 07c391d acd1740 1f932d5 acd1740 07c391d 8322ba8 07c391d 7ff12f8 07c391d 3a56f8d 7ff12f8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | import os
import joblib
import langchain
import streamlit as st
from langchain.chains import RetrievalQAWithSourcesChain
from langchain_community.document_loaders import UnstructuredURLLoader,WebBaseLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.embeddings import SentenceTransformerEmbeddings
from langchain_community.vectorstores import Chroma, FAISS
from langchain_groq import ChatGroq
from dotenv import load_dotenv
import time
load_dotenv("ping.env")
# ✅ Setup LLM (Groq)
llm = ChatGroq(
model="meta-llama/llama-4-scout-17b-16e-instruct",
temperature=0.7,
groq_api_key=os.getenv("GROQ_API_KEY"))
m1=joblib.load("m1.joblib")
st.title("URL ANALYSER🔗")
st.sidebar.title("Give your URls🔗?")
mp=st.empty()
urs=[]
for i in range(3):
url=st.sidebar.text_input(f"URL {i+1}🔗")
urs.append(url)
purs=st.button("gotcha", disabled=not any(url.strip() for url in urs))
if purs:
urs = [url.strip() for url in urs if url.strip()]
mp.text("Loading..URl..Loader....☑️☑️☑️")
valid_urls = [url for url in urs if url.strip()]
if not valid_urls:
st.warning("⚠️ No valid URLs entered.")
st.stop()
try:
sic = WebBaseLoader(valid_urls)
docs = sic.load()
except Exception as e:
st.error(f"❌ Failed to load URLs: {e}")
st.stop()
if not docs:
st.warning("⚠️ No content loaded from URLs. This might be due to network restrictions or invalid URLs.")
st.stop()
st.write(len(docs))
mp.text("Loading..txt..splitter....☑️☑️☑️")
tot=RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=512,chunk_overlap=16)
doccs=tot.split_documents(docs)
mp.text("Loading..VB...☑️☑️☑️")
vv=Chroma.from_documents(doccs,m1)
r2=vv.as_retriever(search_type="similarity",search_kwargs={"k":4})
mp.text("Loading..Retri....☑️☑️☑️")
ra1=RetrievalQAWithSourcesChain.from_chain_type(llm=llm,retriever=r2,chain_type="map_reduce")
st.session_state.ra1=ra1
mp.text("DB & Retri Done ✅✅✅")
time.sleep(3)
query=mp.text_input("UR Question??")
if query:
if "ra1" not in st.session_state:
st.warning("pls give ur urls")
else:
with st.spinner("Wait for it..."):
result=st.session_state.ra1({"question":query},return_only_outputs=True)
st.header("Answer")
st.subheader(result["answer"])
|