Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -312,153 +312,180 @@
|
|
| 312 |
# if __name__ == "__main__":
|
| 313 |
# main()
|
| 314 |
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
import os
|
| 319 |
-
import gradio as gr
|
| 320 |
-
import json
|
| 321 |
import requests
|
|
|
|
|
|
|
| 322 |
from bs4 import BeautifulSoup
|
| 323 |
from sentence_transformers import SentenceTransformer
|
| 324 |
-
import
|
| 325 |
-
from
|
| 326 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
|
| 328 |
-
# --- CONFIG ---
|
| 329 |
-
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
|
| 330 |
groq_client = Groq(api_key=GROQ_API_KEY)
|
| 331 |
embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
|
|
|
|
| 332 |
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
return embed_model.encode([text])[0]
|
| 336 |
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 341 |
results = []
|
| 342 |
-
for
|
| 343 |
-
|
| 344 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 345 |
return results
|
| 346 |
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
for a in links:
|
| 381 |
-
if query.lower() in a.text.lower():
|
| 382 |
-
href = a['href']
|
| 383 |
-
full_url = href if href.startswith("http") else base_url + href
|
| 384 |
-
matching.append(f"{a.text.strip()}\nπ {full_url}")
|
| 385 |
-
if len(matching) >= max_results:
|
| 386 |
-
break
|
| 387 |
-
return matching
|
| 388 |
-
except Exception as e:
|
| 389 |
-
return [f"π Error fetching website: {e}"]
|
| 390 |
-
|
| 391 |
-
def search_youtube_icodeguru(query, max_results=3):
|
| 392 |
-
try:
|
| 393 |
-
api_key = os.getenv("YOUTUBE_API_KEY")
|
| 394 |
-
if not api_key:
|
| 395 |
-
return ["β οΈ YouTube API key not set."]
|
| 396 |
-
|
| 397 |
-
url = f"https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=UCsv3kmQ5k9z7CdFtnqp1U_A&q={query}&type=video&maxResults={max_results}&key={api_key}"
|
| 398 |
-
r = requests.get(url)
|
| 399 |
-
data = r.json()
|
| 400 |
-
|
| 401 |
-
if "items" not in data:
|
| 402 |
-
return ["β οΈ YouTube API error or quota exceeded."]
|
| 403 |
-
|
| 404 |
-
results = []
|
| 405 |
-
for item in data["items"]:
|
| 406 |
-
title = item["snippet"]["title"]
|
| 407 |
-
video_id = item["id"]["videoId"]
|
| 408 |
-
video_url = f"https://www.youtube.com/watch?v={video_id}"
|
| 409 |
-
results.append(f"βΆοΈ {title}\nπ {video_url}")
|
| 410 |
-
|
| 411 |
-
return results
|
| 412 |
-
except Exception as e:
|
| 413 |
-
return [f"πΊ Error searching YouTube: {e}"]
|
| 414 |
-
|
| 415 |
-
# --- CHAT HANDLER ---
|
| 416 |
-
|
| 417 |
-
def chat_handler(message, history, files=None):
|
| 418 |
-
fallback_used = False
|
| 419 |
-
|
| 420 |
-
file_results = search_uploaded_files(message, files) if files else []
|
| 421 |
-
context_text = "\n\n".join(file_results)
|
| 422 |
-
|
| 423 |
-
if not context_text.strip():
|
| 424 |
-
fallback_used = True
|
| 425 |
-
yt_results = search_youtube_icodeguru(message)
|
| 426 |
-
web_results = scrape_icodeguru(message)
|
| 427 |
-
context_text = "\n\n".join(yt_results + web_results)
|
| 428 |
-
|
| 429 |
-
prompt = f"""You are SmartiBot, a helpful assistant trained only on iCodeGuru content.
|
| 430 |
-
Answer the following question **strictly** based on this context:
|
| 431 |
-
|
| 432 |
-
Context:
|
| 433 |
-
{context_text}
|
| 434 |
-
|
| 435 |
-
Question: {message}
|
| 436 |
-
Answer:"""
|
| 437 |
-
|
| 438 |
chat_completion = groq_client.chat.completions.create(
|
| 439 |
model="llama3-8b-8192",
|
| 440 |
-
messages=
|
| 441 |
)
|
|
|
|
| 442 |
|
| 443 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
|
| 445 |
-
|
| 446 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
|
| 448 |
-
|
|
|
|
|
|
|
| 449 |
|
| 450 |
-
|
| 451 |
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
|
|
|
|
|
|
| 461 |
|
| 462 |
if __name__ == "__main__":
|
| 463 |
main()
|
| 464 |
|
|
|
|
|
|
| 312 |
# if __name__ == "__main__":
|
| 313 |
# main()
|
| 314 |
|
| 315 |
+
import nest_asyncio
|
| 316 |
+
import streamlit as st
|
|
|
|
| 317 |
import os
|
|
|
|
|
|
|
| 318 |
import requests
|
| 319 |
+
from youtube_transcript_api import YouTubeTranscriptApi
|
| 320 |
+
from groq import Groq
|
| 321 |
from bs4 import BeautifulSoup
|
| 322 |
from sentence_transformers import SentenceTransformer
|
| 323 |
+
import chromadb
|
| 324 |
+
from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
|
| 325 |
+
|
| 326 |
+
import json
|
| 327 |
+
|
| 328 |
+
nest_asyncio.apply()
|
| 329 |
+
|
| 330 |
+
# --- CONFIGURATION ---
|
| 331 |
+
YOUTUBE_API_KEY = os.environ.get("YOUTUBE_API_KEY")
|
| 332 |
+
GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
|
| 333 |
+
channel_id = "UCsv3kmQ5k1eIRG2R9mWN" # iCodeGuru
|
| 334 |
+
BASE_URL = "https://icode.guru"
|
| 335 |
|
|
|
|
|
|
|
| 336 |
groq_client = Groq(api_key=GROQ_API_KEY)
|
| 337 |
embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
|
| 338 |
+
embedding_function = SentenceTransformerEmbeddingFunction(embed_model)
|
| 339 |
|
| 340 |
+
chroma_client = chromadb.Client()
|
| 341 |
+
collection = chroma_client.get_or_create_collection("icodeguru_knowledge", embedding_function=embedding_function)
|
|
|
|
| 342 |
|
| 343 |
+
# --- Upload + load files as vector DB ---
|
| 344 |
+
def load_uploaded_vectors(uploaded_files):
|
| 345 |
+
data = []
|
| 346 |
+
for file in uploaded_files:
|
| 347 |
+
if file.name.endswith(".txt"):
|
| 348 |
+
text = file.read().decode()
|
| 349 |
+
data.append({"id": file.name, "content": text})
|
| 350 |
+
elif file.name.endswith(".json"):
|
| 351 |
+
content = json.load(file)
|
| 352 |
+
for i, chunk in enumerate(content):
|
| 353 |
+
data.append({"id": f"{file.name}-{i}", "content": chunk})
|
| 354 |
+
return data
|
| 355 |
+
|
| 356 |
+
def search_vector_data(query, data):
|
| 357 |
+
if not data:
|
| 358 |
+
return None
|
| 359 |
+
collection = chroma_client.get_or_create_collection("temp_query", embedding_function=embedding_function)
|
| 360 |
+
collection.add(documents=[d["content"] for d in data], ids=[d["id"] for d in data])
|
| 361 |
+
results = collection.query(query_texts=[query], n_results=3)
|
| 362 |
+
if results and results["documents"]:
|
| 363 |
+
return "\n\n".join([doc for doc in results["documents"][0]])
|
| 364 |
+
return None
|
| 365 |
+
|
| 366 |
+
# --- Fetch recent video IDs from YouTube channel ---
|
| 367 |
+
def get_latest_video_ids(channel_id, max_results=5):
|
| 368 |
+
url = f"https://www.googleapis.com/youtube/v3/search?key={YOUTUBE_API_KEY}&channelId={channel_id}&part=snippet,id&order=date&maxResults={max_results}"
|
| 369 |
+
response = requests.get(url)
|
| 370 |
+
videos = response.json().get('items', [])
|
| 371 |
+
|
| 372 |
+
valid_videos = []
|
| 373 |
+
for v in videos:
|
| 374 |
+
if v['id']['kind'] == 'youtube#video':
|
| 375 |
+
title = v['snippet']['title']
|
| 376 |
+
channel_title = v['snippet']['channelTitle']
|
| 377 |
+
video_id = v['id']['videoId']
|
| 378 |
+
if "icodeguru" in channel_title.lower():
|
| 379 |
+
valid_videos.append((video_id, title))
|
| 380 |
+
return valid_videos
|
| 381 |
+
|
| 382 |
+
# --- Get video transcripts ---
|
| 383 |
+
def get_video_transcripts(video_info):
|
| 384 |
results = []
|
| 385 |
+
for vid, title in video_info:
|
| 386 |
+
try:
|
| 387 |
+
transcript = YouTubeTranscriptApi.get_transcript(vid)
|
| 388 |
+
text = " ".join([t['text'] for t in transcript])
|
| 389 |
+
video_link = f"https://www.youtube.com/watch?v={vid}"
|
| 390 |
+
results.append({
|
| 391 |
+
"video_id": vid,
|
| 392 |
+
"title": title,
|
| 393 |
+
"link": video_link,
|
| 394 |
+
"transcript": text
|
| 395 |
+
})
|
| 396 |
+
except:
|
| 397 |
+
continue
|
| 398 |
return results
|
| 399 |
|
| 400 |
+
# --- Scrape icode.guru ---
|
| 401 |
+
def scrape_icodeguru(base_url=BASE_URL, max_pages=5):
|
| 402 |
+
visited = set()
|
| 403 |
+
blocks = []
|
| 404 |
+
|
| 405 |
+
def crawl(url):
|
| 406 |
+
if url in visited or len(visited) >= max_pages:
|
| 407 |
+
return
|
| 408 |
+
visited.add(url)
|
| 409 |
+
try:
|
| 410 |
+
res = requests.get(url, timeout=10)
|
| 411 |
+
soup = BeautifulSoup(res.content, "html.parser")
|
| 412 |
+
page_text = soup.get_text(separator=" ", strip=True)
|
| 413 |
+
if len(page_text) > 100:
|
| 414 |
+
blocks.append(f"[{url}]({url}):\n{page_text[:1500]}")
|
| 415 |
+
for link in soup.find_all("a", href=True):
|
| 416 |
+
href = link['href']
|
| 417 |
+
if href.startswith("/"):
|
| 418 |
+
href = base_url + href
|
| 419 |
+
if href.startswith(base_url):
|
| 420 |
+
crawl(href)
|
| 421 |
+
except:
|
| 422 |
+
pass
|
| 423 |
+
|
| 424 |
+
crawl(base_url)
|
| 425 |
+
return blocks
|
| 426 |
+
|
| 427 |
+
# --- Ask Groq ---
|
| 428 |
+
def ask_groq(context, question):
|
| 429 |
+
messages = [
|
| 430 |
+
{"role": "system", "content": "You are a helpful assistant. Always provide relevant video and website links if possible."},
|
| 431 |
+
{"role": "user", "content": f"Context:\n{context}\n\nQuestion: {question}\nAnswer (include links):"}
|
| 432 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
chat_completion = groq_client.chat.completions.create(
|
| 434 |
model="llama3-8b-8192",
|
| 435 |
+
messages=messages,
|
| 436 |
)
|
| 437 |
+
return chat_completion.choices[0].message.content.strip()
|
| 438 |
|
| 439 |
+
#--- STREAMLIT APP ---
|
| 440 |
+
def main():
|
| 441 |
+
st.set_page_config(page_title="EduBot for iCodeGuru", layout="wide")
|
| 442 |
+
st.title("π EduBot for @icodeguru0")
|
| 443 |
+
st.markdown("Ask anything based on the latest YouTube videos and website content of [icode.guru](https://icode.guru).")
|
| 444 |
+
|
| 445 |
+
uploaded_files = st.file_uploader("π Optionally upload your knowledge files (txt or json)", type=['txt', 'json'], accept_multiple_files=True)
|
| 446 |
+
user_question = st.text_input("π¬ Ask your question:")
|
| 447 |
+
|
| 448 |
+
if user_question:
|
| 449 |
+
vector_data = load_uploaded_vectors(uploaded_files) if uploaded_files else []
|
| 450 |
+
|
| 451 |
+
# Try vector DB first
|
| 452 |
+
vector_context = search_vector_data(user_question, vector_data)
|
| 453 |
+
if vector_context:
|
| 454 |
+
with st.spinner("π§ Answering from uploaded knowledge..."):
|
| 455 |
+
answer = ask_groq(vector_context, user_question)
|
| 456 |
+
st.success(answer)
|
| 457 |
+
else:
|
| 458 |
+
# Fallback to real-time data
|
| 459 |
+
with st.spinner("πΊ Fetching YouTube videos..."):
|
| 460 |
+
video_info = get_latest_video_ids(channel_id, max_results=5)
|
| 461 |
+
transcripts = get_video_transcripts(video_info)
|
| 462 |
|
| 463 |
+
yt_context = ""
|
| 464 |
+
relevant_links = []
|
| 465 |
+
for vid in transcripts:
|
| 466 |
+
yt_context += f"\n\n[Video: {vid['title']}]({vid['link']}):\n{vid['transcript'][:1500]}"
|
| 467 |
+
if user_question.lower() in vid['transcript'].lower():
|
| 468 |
+
relevant_links.append(vid['link'])
|
| 469 |
|
| 470 |
+
with st.spinner("π Scraping icode.guru..."):
|
| 471 |
+
site_blocks = scrape_icodeguru(BASE_URL, max_pages=5)
|
| 472 |
+
site_context = "\n\n".join(site_blocks)
|
| 473 |
|
| 474 |
+
full_context = yt_context + "\n\n" + site_context
|
| 475 |
|
| 476 |
+
with st.spinner("π§ Thinking..."):
|
| 477 |
+
answer = ask_groq(full_context, user_question)
|
| 478 |
+
st.success(answer)
|
| 479 |
+
|
| 480 |
+
if relevant_links:
|
| 481 |
+
st.markdown("### π Related YouTube Links")
|
| 482 |
+
for link in relevant_links:
|
| 483 |
+
st.markdown(f"- [Watch Video]({link})")
|
| 484 |
+
|
| 485 |
+
st.markdown("---")
|
| 486 |
+
st.caption("Powered by YouTube, iCodeGuru, and Groq")
|
| 487 |
|
| 488 |
if __name__ == "__main__":
|
| 489 |
main()
|
| 490 |
|
| 491 |
+
|