ilsa15 commited on
Commit
96a0876
Β·
verified Β·
1 Parent(s): 4ef48cd

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +153 -126
app.py CHANGED
@@ -312,153 +312,180 @@
312
  # if __name__ == "__main__":
313
  # main()
314
 
315
-
316
-
317
-
318
  import os
319
- import gradio as gr
320
- import json
321
  import requests
 
 
322
  from bs4 import BeautifulSoup
323
  from sentence_transformers import SentenceTransformer
324
- import numpy as np
325
- from sklearn.metrics.pairwise import cosine_similarity
326
- from groq import Groq
 
 
 
 
 
 
 
 
 
327
 
328
- # --- CONFIG ---
329
- GROQ_API_KEY = os.getenv("GROQ_API_KEY")
330
  groq_client = Groq(api_key=GROQ_API_KEY)
331
  embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
332
 
333
- # --- UTILS ---
334
- def embed(text):
335
- return embed_model.encode([text])[0]
336
 
337
- def find_best_match(query, documents, embeddings, top_k=3, threshold=0.5):
338
- query_vec = embed(query).reshape(1, -1)
339
- sims = cosine_similarity(query_vec, embeddings)[0]
340
- top_indices = np.argsort(sims)[::-1]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
341
  results = []
342
- for idx in top_indices[:top_k]:
343
- if sims[idx] >= threshold:
344
- results.append((documents[idx], float(sims[idx])))
 
 
 
 
 
 
 
 
 
 
345
  return results
346
 
347
- def search_uploaded_files(query, uploaded_files):
348
- texts = []
349
- sources = []
350
-
351
- for file in uploaded_files:
352
- name = file.name.lower()
353
- content = file.read().decode("utf-8")
354
- if name.endswith(".json"):
355
- data = json.loads(content)
356
- for item in data if isinstance(data, list) else [data]:
357
- text = json.dumps(item) if isinstance(item, dict) else str(item)
358
- texts.append(text)
359
- sources.append(name)
360
- else:
361
- texts.append(content)
362
- sources.append(name)
363
-
364
- if not texts:
365
- return []
366
-
367
- embeddings = [embed(t) for t in texts]
368
- top_results = find_best_match(query, texts, embeddings)
369
- return [f"πŸ“„ From {sources[i]}:\n{text}" for i, (text, _) in enumerate(top_results)]
370
-
371
- # --- FALLBACK SOURCES ---
372
-
373
- def scrape_icodeguru(query, max_results=3):
374
- base_url = "https://icode.guru"
375
- try:
376
- resp = requests.get(base_url, timeout=10)
377
- soup = BeautifulSoup(resp.text, "html.parser")
378
- links = soup.find_all("a", href=True)
379
- matching = []
380
- for a in links:
381
- if query.lower() in a.text.lower():
382
- href = a['href']
383
- full_url = href if href.startswith("http") else base_url + href
384
- matching.append(f"{a.text.strip()}\nπŸ”— {full_url}")
385
- if len(matching) >= max_results:
386
- break
387
- return matching
388
- except Exception as e:
389
- return [f"🌐 Error fetching website: {e}"]
390
-
391
- def search_youtube_icodeguru(query, max_results=3):
392
- try:
393
- api_key = os.getenv("YOUTUBE_API_KEY")
394
- if not api_key:
395
- return ["⚠️ YouTube API key not set."]
396
-
397
- url = f"https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=UCsv3kmQ5k9z7CdFtnqp1U_A&q={query}&type=video&maxResults={max_results}&key={api_key}"
398
- r = requests.get(url)
399
- data = r.json()
400
-
401
- if "items" not in data:
402
- return ["⚠️ YouTube API error or quota exceeded."]
403
-
404
- results = []
405
- for item in data["items"]:
406
- title = item["snippet"]["title"]
407
- video_id = item["id"]["videoId"]
408
- video_url = f"https://www.youtube.com/watch?v={video_id}"
409
- results.append(f"▢️ {title}\nπŸ”— {video_url}")
410
-
411
- return results
412
- except Exception as e:
413
- return [f"πŸ“Ί Error searching YouTube: {e}"]
414
-
415
- # --- CHAT HANDLER ---
416
-
417
- def chat_handler(message, history, files=None):
418
- fallback_used = False
419
-
420
- file_results = search_uploaded_files(message, files) if files else []
421
- context_text = "\n\n".join(file_results)
422
-
423
- if not context_text.strip():
424
- fallback_used = True
425
- yt_results = search_youtube_icodeguru(message)
426
- web_results = scrape_icodeguru(message)
427
- context_text = "\n\n".join(yt_results + web_results)
428
-
429
- prompt = f"""You are SmartiBot, a helpful assistant trained only on iCodeGuru content.
430
- Answer the following question **strictly** based on this context:
431
-
432
- Context:
433
- {context_text}
434
-
435
- Question: {message}
436
- Answer:"""
437
-
438
  chat_completion = groq_client.chat.completions.create(
439
  model="llama3-8b-8192",
440
- messages=[{"role": "user", "content": prompt}],
441
  )
 
442
 
443
- reply = chat_completion.choices[0].message.content.strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
444
 
445
- if fallback_used:
446
- reply += "\n\nπŸ“‘ Fetched from iCodeGuru YouTube or Website."
 
 
 
 
447
 
448
- return reply
 
 
449
 
450
- # --- STREAMLIT APP ---
451
 
452
- def main():
453
- gr.ChatInterface(
454
- fn=chat_handler,
455
- title="πŸ€– SmartiBot for iCodeGuru",
456
- description="Ask any tech question β€” get answers only from iCodeGuru's YouTube, Website, or uploaded files!",
457
- additional_inputs=[
458
- gr.File(file_types=[".txt", ".json"], label="Upload Knowledge Files", file_count="multiple")
459
- ],
460
- ).launch()
 
 
461
 
462
  if __name__ == "__main__":
463
  main()
464
 
 
 
312
  # if __name__ == "__main__":
313
  # main()
314
 
315
+ import nest_asyncio
316
+ import streamlit as st
 
317
  import os
 
 
318
  import requests
319
+ from youtube_transcript_api import YouTubeTranscriptApi
320
+ from groq import Groq
321
  from bs4 import BeautifulSoup
322
  from sentence_transformers import SentenceTransformer
323
+ import chromadb
324
+ from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
325
+
326
+ import json
327
+
328
+ nest_asyncio.apply()
329
+
330
+ # --- CONFIGURATION ---
331
+ YOUTUBE_API_KEY = os.environ.get("YOUTUBE_API_KEY")
332
+ GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
333
+ channel_id = "UCsv3kmQ5k1eIRG2R9mWN" # iCodeGuru
334
+ BASE_URL = "https://icode.guru"
335
 
 
 
336
  groq_client = Groq(api_key=GROQ_API_KEY)
337
  embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
338
+ embedding_function = SentenceTransformerEmbeddingFunction(embed_model)
339
 
340
+ chroma_client = chromadb.Client()
341
+ collection = chroma_client.get_or_create_collection("icodeguru_knowledge", embedding_function=embedding_function)
 
342
 
343
+ # --- Upload + load files as vector DB ---
344
+ def load_uploaded_vectors(uploaded_files):
345
+ data = []
346
+ for file in uploaded_files:
347
+ if file.name.endswith(".txt"):
348
+ text = file.read().decode()
349
+ data.append({"id": file.name, "content": text})
350
+ elif file.name.endswith(".json"):
351
+ content = json.load(file)
352
+ for i, chunk in enumerate(content):
353
+ data.append({"id": f"{file.name}-{i}", "content": chunk})
354
+ return data
355
+
356
+ def search_vector_data(query, data):
357
+ if not data:
358
+ return None
359
+ collection = chroma_client.get_or_create_collection("temp_query", embedding_function=embedding_function)
360
+ collection.add(documents=[d["content"] for d in data], ids=[d["id"] for d in data])
361
+ results = collection.query(query_texts=[query], n_results=3)
362
+ if results and results["documents"]:
363
+ return "\n\n".join([doc for doc in results["documents"][0]])
364
+ return None
365
+
366
+ # --- Fetch recent video IDs from YouTube channel ---
367
+ def get_latest_video_ids(channel_id, max_results=5):
368
+ url = f"https://www.googleapis.com/youtube/v3/search?key={YOUTUBE_API_KEY}&channelId={channel_id}&part=snippet,id&order=date&maxResults={max_results}"
369
+ response = requests.get(url)
370
+ videos = response.json().get('items', [])
371
+
372
+ valid_videos = []
373
+ for v in videos:
374
+ if v['id']['kind'] == 'youtube#video':
375
+ title = v['snippet']['title']
376
+ channel_title = v['snippet']['channelTitle']
377
+ video_id = v['id']['videoId']
378
+ if "icodeguru" in channel_title.lower():
379
+ valid_videos.append((video_id, title))
380
+ return valid_videos
381
+
382
+ # --- Get video transcripts ---
383
+ def get_video_transcripts(video_info):
384
  results = []
385
+ for vid, title in video_info:
386
+ try:
387
+ transcript = YouTubeTranscriptApi.get_transcript(vid)
388
+ text = " ".join([t['text'] for t in transcript])
389
+ video_link = f"https://www.youtube.com/watch?v={vid}"
390
+ results.append({
391
+ "video_id": vid,
392
+ "title": title,
393
+ "link": video_link,
394
+ "transcript": text
395
+ })
396
+ except:
397
+ continue
398
  return results
399
 
400
+ # --- Scrape icode.guru ---
401
+ def scrape_icodeguru(base_url=BASE_URL, max_pages=5):
402
+ visited = set()
403
+ blocks = []
404
+
405
+ def crawl(url):
406
+ if url in visited or len(visited) >= max_pages:
407
+ return
408
+ visited.add(url)
409
+ try:
410
+ res = requests.get(url, timeout=10)
411
+ soup = BeautifulSoup(res.content, "html.parser")
412
+ page_text = soup.get_text(separator=" ", strip=True)
413
+ if len(page_text) > 100:
414
+ blocks.append(f"[{url}]({url}):\n{page_text[:1500]}")
415
+ for link in soup.find_all("a", href=True):
416
+ href = link['href']
417
+ if href.startswith("/"):
418
+ href = base_url + href
419
+ if href.startswith(base_url):
420
+ crawl(href)
421
+ except:
422
+ pass
423
+
424
+ crawl(base_url)
425
+ return blocks
426
+
427
+ # --- Ask Groq ---
428
+ def ask_groq(context, question):
429
+ messages = [
430
+ {"role": "system", "content": "You are a helpful assistant. Always provide relevant video and website links if possible."},
431
+ {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {question}\nAnswer (include links):"}
432
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
433
  chat_completion = groq_client.chat.completions.create(
434
  model="llama3-8b-8192",
435
+ messages=messages,
436
  )
437
+ return chat_completion.choices[0].message.content.strip()
438
 
439
+ #--- STREAMLIT APP ---
440
+ def main():
441
+ st.set_page_config(page_title="EduBot for iCodeGuru", layout="wide")
442
+ st.title("πŸŽ“ EduBot for @icodeguru0")
443
+ st.markdown("Ask anything based on the latest YouTube videos and website content of [icode.guru](https://icode.guru).")
444
+
445
+ uploaded_files = st.file_uploader("πŸ“ Optionally upload your knowledge files (txt or json)", type=['txt', 'json'], accept_multiple_files=True)
446
+ user_question = st.text_input("πŸ’¬ Ask your question:")
447
+
448
+ if user_question:
449
+ vector_data = load_uploaded_vectors(uploaded_files) if uploaded_files else []
450
+
451
+ # Try vector DB first
452
+ vector_context = search_vector_data(user_question, vector_data)
453
+ if vector_context:
454
+ with st.spinner("🧠 Answering from uploaded knowledge..."):
455
+ answer = ask_groq(vector_context, user_question)
456
+ st.success(answer)
457
+ else:
458
+ # Fallback to real-time data
459
+ with st.spinner("πŸ“Ί Fetching YouTube videos..."):
460
+ video_info = get_latest_video_ids(channel_id, max_results=5)
461
+ transcripts = get_video_transcripts(video_info)
462
 
463
+ yt_context = ""
464
+ relevant_links = []
465
+ for vid in transcripts:
466
+ yt_context += f"\n\n[Video: {vid['title']}]({vid['link']}):\n{vid['transcript'][:1500]}"
467
+ if user_question.lower() in vid['transcript'].lower():
468
+ relevant_links.append(vid['link'])
469
 
470
+ with st.spinner("🌐 Scraping icode.guru..."):
471
+ site_blocks = scrape_icodeguru(BASE_URL, max_pages=5)
472
+ site_context = "\n\n".join(site_blocks)
473
 
474
+ full_context = yt_context + "\n\n" + site_context
475
 
476
+ with st.spinner("🧠 Thinking..."):
477
+ answer = ask_groq(full_context, user_question)
478
+ st.success(answer)
479
+
480
+ if relevant_links:
481
+ st.markdown("### πŸ”— Related YouTube Links")
482
+ for link in relevant_links:
483
+ st.markdown(f"- [Watch Video]({link})")
484
+
485
+ st.markdown("---")
486
+ st.caption("Powered by YouTube, iCodeGuru, and Groq")
487
 
488
  if __name__ == "__main__":
489
  main()
490
 
491
+