File size: 2,531 Bytes
8d29339
5c312ac
7ff12f8
 
e24fd2c
7ff12f8
1f932d5
57232db
8d29339
 
3d5d389
7ff12f8
 
8d29339
2a6d4e6
7ff12f8
3d5d389
 
 
 
f6fff89
3d5d389
f26622c
7ff12f8
1aa1cba
7ff12f8
 
 
 
acd1740
 
 
 
7ff12f8
0395bcc
7ff12f8
acd1740
7ff12f8
cb638c9
acd1740
8322ba8
07c391d
acd1740
 
 
 
 
1f932d5
acd1740
 
 
 
 
 
 
 
 
07c391d
 
 
8322ba8
07c391d
 
 
 
 
 
 
 
7ff12f8
 
07c391d
 
 
 
 
 
 
3a56f8d
7ff12f8
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import os
import joblib
import langchain
import streamlit as st

from langchain.chains import RetrievalQAWithSourcesChain
from langchain_community.document_loaders import UnstructuredURLLoader,WebBaseLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.embeddings import SentenceTransformerEmbeddings
from langchain_community.vectorstores import Chroma, FAISS
from langchain_groq import ChatGroq
from dotenv import load_dotenv
import time

load_dotenv("ping.env")

# ✅ Setup LLM (Groq)
llm = ChatGroq(
    model="meta-llama/llama-4-scout-17b-16e-instruct",
    temperature=0.7,
    groq_api_key=os.getenv("GROQ_API_KEY"))
    


m1=joblib.load("m1.joblib")
st.title("URL ANALYSER🔗")
st.sidebar.title("Give your URls🔗?")
mp=st.empty()

urs=[]
for i in range(3):
  url=st.sidebar.text_input(f"URL {i+1}🔗")
  urs.append(url)



purs=st.button("gotcha", disabled=not any(url.strip() for url in urs))
if purs:
    
    urs = [url.strip() for url in urs if url.strip()]
    
    mp.text("Loading..URl..Loader....☑️☑️☑️")
    valid_urls = [url for url in urs if url.strip()]
    if not valid_urls:
        st.warning("⚠️ No valid URLs entered.")
        st.stop()
    try:
        sic = WebBaseLoader(valid_urls)
        docs = sic.load()
    except Exception as e:
        st.error(f"❌ Failed to load URLs: {e}")
        st.stop()
    if not docs:
        st.warning("⚠️ No content loaded from URLs. This might be due to network restrictions or invalid URLs.")
        st.stop()
    st.write(len(docs))

    mp.text("Loading..txt..splitter....☑️☑️☑️")
    tot=RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=512,chunk_overlap=16)
    doccs=tot.split_documents(docs)
    
    mp.text("Loading..VB...☑️☑️☑️")
    vv=Chroma.from_documents(doccs,m1)
    r2=vv.as_retriever(search_type="similarity",search_kwargs={"k":4})
    mp.text("Loading..Retri....☑️☑️☑️")
    ra1=RetrievalQAWithSourcesChain.from_chain_type(llm=llm,retriever=r2,chain_type="map_reduce")
    st.session_state.ra1=ra1
    mp.text("DB & Retri Done ✅✅✅")
    time.sleep(3)
query=mp.text_input("UR Question??")
if query:
    if "ra1" not in st.session_state:
        st.warning("pls give ur urls")
    else:
        with st.spinner("Wait for it..."):
            result=st.session_state.ra1({"question":query},return_only_outputs=True)
        st.header("Answer")
        st.subheader(result["answer"])