import csv
import json
import os
import re
import time
import xml.etree.ElementTree as ET
from urllib.parse import parse_qs, urlparse
from bs4 import BeautifulSoup
import docx
import gradio as gr
from langchain_community.vectorstores import FAISS
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from langchain_huggingface import (
ChatHuggingFace,
HuggingFaceEmbeddings,
HuggingFaceEndpoint,
)
import nltk
from nltk.tokenize import word_tokenize
import openpyxl
import pptx
import PyPDF2
import requests
from youtube_transcript_api import YouTubeTranscriptApi
from youtube_transcript_api._errors import (
NoTranscriptFound,
TranscriptsDisabled,
VideoUnavailable,
)
nltk.download("punkt")
nltk.download("punkt_tab")
nltk.download("omw-1.4")
nltk.download("wordnet")
def read_csv(file_path):
with open(
file_path,
"r",
encoding="utf-8",
errors="ignore",
newline=""
) as csvfile:
csv_reader = csv.reader(csvfile)
csv_data = [row for row in csv_reader]
return " ".join(
[" ".join(row) for row in csv_data]
)
def read_text(file_path):
with open(
file_path,
"r",
encoding="utf-8",
errors="ignore"
) as f:
return f.read()
def read_pdf(file_path):
text_data = []
with open(file_path, "rb") as pdf_file:
pdf_reader = PyPDF2.PdfReader(pdf_file)
for page in pdf_reader.pages:
page_text = page.extract_text()
if page_text:
text_data.append(page_text)
return "\n".join(text_data)
def read_docx(file_path):
doc = docx.Document(file_path)
return "\n".join(
paragraph.text
for paragraph in doc.paragraphs
)
def read_pptx(file_path):
ppt = pptx.Presentation(file_path)
text_data = ""
for slide in ppt.slides:
for shape in slide.shapes:
if hasattr(shape, "text"):
text_data += shape.text + "\n"
return text_data
def read_xlsx(file_path):
workbook = openpyxl.load_workbook(file_path)
sheet = workbook.active
text_data = ""
for row in sheet.iter_rows(values_only=True):
text_data += (
" ".join(
str(cell)
for cell in row
if cell is not None
)
+ "\n"
)
return text_data
def read_json(file_path):
with open(
file_path,
"r",
encoding="utf-8"
) as f:
json_data = json.load(f)
return json.dumps(
json_data,
ensure_ascii=False
)
def read_html(file_path):
with open(
file_path,
"r",
encoding="utf-8",
errors="ignore"
) as f:
html_content = f.read()
soup = BeautifulSoup(
html_content,
"html.parser"
)
return soup.get_text(
separator="\n"
)
def read_xml(file_path):
tree = ET.parse(file_path)
root = tree.getroot()
return ET.tostring(
root,
encoding="unicode"
)
def extract_youtube_video_id(url):
if not url:
return None
url = str(url).strip()
url = url.replace("\\", "")
url = url.replace("\n", "")
url = url.replace("\\n", "")
markdown_match = re.search(
r"\]\(\s*(https?://[^)\s]+)",
url,
flags=re.IGNORECASE
)
if markdown_match:
url = markdown_match.group(1)
url_match = re.search(
r"https?://(?:www\.)?(?:youtube\.com|youtu\.be)[^\s<>\"']+",
url,
flags=re.IGNORECASE
)
if url_match:
url = url_match.group(0)
url = url.rstrip(
".,!?;:)]}"
)
parsed = urlparse(url)
hostname = (
parsed.hostname or ""
).lower()
if hostname in (
"youtu.be",
"www.youtu.be"
):
video_id = (
parsed.path
.lstrip("/")
.split("/")[0]
)
return (
video_id
.replace("\\", "")
.strip()
or None
)
if hostname in (
"youtube.com",
"www.youtube.com",
"m.youtube.com"
):
if parsed.path == "/watch":
video_ids = parse_qs(
parsed.query
).get("v")
if video_ids:
video_id = (
video_ids[0]
.replace("\\", "")
.strip()
)
return video_id or None
for prefix in (
"/shorts/",
"/embed/",
"/live/"
):
if parsed.path.startswith(prefix):
video_id = (
parsed.path[
len(prefix):
]
.split("/")[0]
.replace("\\", "")
.strip()
)
return video_id or None
return None
def is_youtube_url(url):
return (
extract_youtube_video_id(url)
is not None
)
def fetch_transcript_text(video_id):
video_id = (
str(video_id)
.replace("\\", "")
.strip()
)
print(
"CLEAN YOUTUBE VIDEO ID:",
repr(video_id)
)
try:
print(
"FETCHING YOUTUBE TRANSCRIPT:",
repr(video_id)
)
api = YouTubeTranscriptApi()
transcript_list = api.list(
video_id
)
transcript = None
try:
transcript = (
transcript_list.find_transcript(
["en"]
)
)
except Exception:
try:
transcript = (
transcript_list.find_transcript(
["ar"]
)
)
except Exception:
for item in transcript_list:
transcript = item
break
if transcript is None:
raise NoTranscriptFound(
video_id,
["en", "ar"],
transcript_list
)
fetched = transcript.fetch()
if hasattr(
fetched,
"snippets"
):
text = " ".join(
snippet.text
for snippet in fetched.snippets
)
else:
text = " ".join(
snippet.text
for snippet in fetched
)
text = text.strip()
if not text:
raise RuntimeError(
"Transcript was retrieved "
"but contains no text."
)
print(
"YOUTUBE TRANSCRIPT SUCCESS:",
len(text),
"characters"
)
return text
except (
NoTranscriptFound,
TranscriptsDisabled,
VideoUnavailable
):
raise
except Exception as e:
print(
"YOUTUBE API ERROR:",
repr(e)
)
raise
def process_youtube_video(url):
video_id = extract_youtube_video_id(
url
)
if not video_id:
return (
"Invalid YouTube video URL. "
"Please provide a valid YouTube "
"video link."
)
print(
"YOUTUBE VIDEO ID:",
repr(video_id)
)
try:
transcript = fetch_transcript_text(
video_id
)
if transcript:
return transcript
return (
"The YouTube transcript was retrieved "
"but contains no readable text."
)
except NoTranscriptFound:
return (
"No English or Arabic transcript "
"was found for this YouTube video."
)
except TranscriptsDisabled:
return (
"Transcripts are disabled for this "
"YouTube video."
)
except VideoUnavailable:
return (
"The YouTube video is unavailable."
)
except Exception as e:
print(
"YOUTUBE TRANSCRIPT ERROR:",
repr(e)
)
return (
"Unable to retrieve the YouTube "
"transcript.\n\n"
f"Error: {str(e)}"
)
def read_web_page(url):
try:
url = str(url).strip()
response = requests.get(
url,
headers={
"User-Agent": (
"Mozilla/5.0 "
"(Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 "
"(KHTML, like Gecko) "
"Chrome/151.0.0.0 "
"Safari/537.36"
)
},
timeout=20,
allow_redirects=True
)
if response.status_code >= 400:
return (
"Unable to access webpage. "
f"HTTP status: {response.status_code}"
)
content_type = (
response.headers
.get("content-type", "")
.lower()
)
if "text/plain" in content_type:
return response.text
soup = BeautifulSoup(
response.text,
"html.parser"
)
for element in soup([
"script",
"style",
"noscript",
"svg"
]):
element.decompose()
text_data = soup.get_text(
separator="\n"
)
text_data = "\n".join(
line.strip()
for line in text_data.splitlines()
if line.strip()
)
if not text_data:
return (
"The webpage was accessed successfully "
"but contains no readable text."
)
return text_data
except requests.exceptions.Timeout:
return (
"The webpage request timed out. "
"Please try another link."
)
except requests.exceptions.RequestException as e:
return (
f"Unable to access webpage: {e}"
)
except Exception as e:
return (
"An error occurred while reading "
f"the webpage: {e}"
)
def read_data(
file_path_or_url,
languages=["en", "ar"]
):
if not file_path_or_url:
return "Unsupported type or format."
file_path_or_url = str(
file_path_or_url
).strip()
if is_youtube_url(
file_path_or_url
):
return process_youtube_video(
file_path_or_url
)
if file_path_or_url.startswith(
("http://", "https://")
):
return read_web_page(
file_path_or_url
)
lower_path = (
file_path_or_url.lower()
)
if lower_path.endswith(".csv"):
return read_csv(
file_path_or_url
)
elif lower_path.endswith(".txt"):
return read_text(
file_path_or_url
)
elif lower_path.endswith(".pdf"):
return read_pdf(
file_path_or_url
)
elif lower_path.endswith(".docx"):
return read_docx(
file_path_or_url
)
elif lower_path.endswith(".pptx"):
return read_pptx(
file_path_or_url
)
elif lower_path.endswith(".xlsx"):
return read_xlsx(
file_path_or_url
)
elif lower_path.endswith(".json"):
return read_json(
file_path_or_url
)
elif lower_path.endswith(".html"):
return read_html(
file_path_or_url
)
elif lower_path.endswith(".xml"):
return read_xml(
file_path_or_url
)
return "Unsupported type or format."
def normalize_text(text):
if not isinstance(
text,
str
):
text = str(text)
text = re.sub(
r"\\n",
" ",
text
)
text = re.sub(
r"\\",
"",
text
)
text = text.lower()
text = text.strip()
punctuation = (
r"""!()[]{};:'"\<>/?$%^&*_`~="""
)
for punc in punctuation:
text = text.replace(
punc,
""
)
text = re.sub(
r"[A-Za-z0-9]*@[A-Za-z]*\.?[A-Za-z0-9]*",
"",
text
)
words = word_tokenize(
text
)
return " ".join(words)
llm = HuggingFaceEndpoint(
repo_id="Qwen/Qwen2.5-Coder-32B-Instruct",
task="text-generation",
max_new_tokens=4096,
temperature=0.6,
top_p=0.9,
top_k=40,
repetition_penalty=1.2,
do_sample=True,
)
chat_model = ChatHuggingFace(
llm=llm
)
model_name = (
"sentence-transformers/all-mpnet-base-v2"
)
embedding_llm = HuggingFaceEmbeddings(
model_name=model_name
)
db = FAISS.load_local(
"faiss_index",
embedding_llm,
allow_dangerous_deserialization=True
)
def print_like_dislike(
x: gr.LikeData
):
print(
x.index,
x.value,
x.liked
)
def user(
user_message,
history
):
if not len(user_message):
raise gr.Error(
"Chat messages cannot be empty"
)
history.append({
"role": "user",
"content": user_message
})
return "", history
def user2(
user_message,
history,
link
):
if (
not len(user_message)
or not len(link)
):
raise gr.Error(
"Chat messages or links cannot be empty"
)
link = str(link).strip()
user_message = str(
user_message
).strip()
combined_message = (
f"URL: {link}\n"
f"QUESTION: {user_message}"
)
history.append({
"role": "user",
"content": combined_message
})
return "", history, link
def user3(
user_message,
history,
file_path
):
if (
not len(user_message)
or not file_path
):
raise gr.Error(
"Chat messages or files cannot be empty"
)
combined_message = (
f"{file_path}\n"
f"{user_message}"
)
history.append({
"role": "user",
"content": combined_message
})
return "", history, file_path
messages1_state = [
SystemMessage(
content="You are a helpful assistant."
),
HumanMessage(
content="Hi AI, how are you today?"
),
AIMessage(
content=(
"I'm great thank you. "
"How can I help you?"
)
),
]
def Chat_Message(
history,
messages1
):
user_msg_text = (
history[-1]["content"]
)
message = HumanMessage(
content=user_msg_text
)
if isinstance(
messages1[-1],
HumanMessage
):
messages1 = messages1[:-2]
messages1.append(
message
)
if len(messages1) >= 8:
messages1 = messages1[-8:]
try:
response = chat_model.invoke(
messages1
)
except Exception as e:
error_message = str(e)
print(
"CHAT ERROR:",
repr(e)
)
raise gr.Error(
"Error occurred during response"
) from e
messages1.append(
AIMessage(
content=response.content
)
)
history.append({
"role": "assistant",
"content": ""
})
for character in response.content:
history[-1]["content"] += character
time.sleep(
0.0025
)
yield history, messages1
def Internet_Search(
history,
messages2
):
message = str(
history[-1]["content"]
)
if isinstance(
messages2[-1],
HumanMessage
):
messages2 = messages2[:-2]
similar_docs = db.similarity_search(
message,
k=3
)
if similar_docs:
source_knowledge = "\n".join(
[
x.page_content
for x in similar_docs
]
)
else:
source_knowledge = ""
augmented_prompt = f"""
You are an AI designed to help understand
and extract information from provided Search Content.
Based on the user's Query, you may need to summarize
the text, answer specific questions, or provide guidance.
Query:
{message}
Search Content:
{source_knowledge}
If the query is not related to specific Search Content,
engage in general conversation or provide relevant
information from other sources.
"""
msg = HumanMessage(
content=augmented_prompt
)
messages2.append(
msg
)
if len(messages2) >= 4:
messages2 = messages2[-4:]
try:
response = chat_model.invoke(
messages2
)
except Exception as e:
print(
"INTERNET SEARCH ERROR:",
repr(e)
)
raise gr.Error(
"Error occurred during response"
) from e
messages2.append(
AIMessage(
content=response.content
)
)
history.append({
"role": "assistant",
"content": ""
})
for character in response.content:
history[-1]["content"] += character
time.sleep(
0.0025
)
yield history, messages2
def generate_chart_config(
description
):
system_instructions = """
You are a Chart.js configuration generator.
Return ONLY valid JSON.
The JSON must have this structure:
{
"type": "bar",
"data": {
"labels": [],
"datasets": [
{
"label": "",
"data": []
}
]
},
"options": {}
}
Rules:
- Output JSON only.
- No markdown.
- No code fences.
- No explanation.
- Use valid JSON double quotes.
- Do not use trailing commas.
- The top-level object must contain "type".
- The top-level object must contain "data".
- "data" must contain "labels".
- "data" must contain "datasets".
- The chart must be valid for Chart.js.
"""
prompt = [
SystemMessage(
content=system_instructions
),
HumanMessage(
content=(
"Create a Chart.js chart for "
"this request:\n"
f"{description}"
)
),
]
print(
"CHART DESCRIPTION:",
description
)
response = chat_model.invoke(
prompt
)
raw = str(
response.content
).strip()
print(
"RAW CHART MODEL RESPONSE:",
raw
)
raw = re.sub(
r"^```json\s*",
"",
raw,
flags=re.IGNORECASE
)
raw = re.sub(
r"^```\s*",
"",
raw
)
raw = re.sub(
r"\s*```$",
"",
raw
)
raw = raw.strip()
start = raw.find("{")
end = raw.rfind("}")
if (
start == -1
or end == -1
):
raise ValueError(
"Model did not return a JSON object.\n"
f"Raw response:\n{raw}"
)
raw = raw[
start:end + 1
]
config = json.loads(
raw
)
if "type" not in config:
raise ValueError(
"Chart config missing 'type'"
)
if "data" not in config:
raise ValueError(
"Chart config missing 'data'"
)
if "labels" not in config["data"]:
raise ValueError(
"Chart config missing 'data.labels'"
)
if "datasets" not in config["data"]:
raise ValueError(
"Chart config missing 'data.datasets'"
)
if not isinstance(
config["data"]["datasets"],
list
):
raise ValueError(
"'data.datasets' must be a list"
)
return config
def Chart_Generator(
history,
messages3
):
message = str(
history[-1]["content"]
)
if isinstance(
messages3[-1],
HumanMessage
):
messages3 = messages3[:-2]
if "#chart" in message.lower():
chart_description = re.split(
r"#chart",
message,
maxsplit=1,
flags=re.IGNORECASE
)[1].strip()
if not chart_description:
combined_content = (
"Please provide chart details "
"after #chart."
)
else:
chart_config = None
try:
chart_config = (
generate_chart_config(
chart_description
)
)
except Exception as e:
print(
"CHART GENERATION ERROR:",
repr(e)
)
combined_content = (
"Chart generation failed.\n\n"
f"Error: {str(e)}"
)
if chart_config:
try:
config_json = json.dumps(
chart_config,
separators=(",", ":")
)
encoded_config = (
requests.utils.quote(
config_json,
safe=""
)
)
chart_url = (
"https://quickchart.io/chart"
f"?c={encoded_config}"
"&bkg=white"
)
chart_response = requests.get(
chart_url,
timeout=30
)
if (
chart_response.status_code
!= 200
):
combined_content = (
"QuickChart failed to "
"generate the chart.\n\n"
f"HTTP Status: "
f"{chart_response.status_code}"
)
else:
image_html = (
f''
)
chart_summary_prompt = (
"The following Chart.js "
"configuration was generated:\n\n"
f"{json.dumps(chart_config, indent=2)}\n\n"
"Briefly describe what this chart "
"represents."
)
analysis_messages = [
SystemMessage(
content=(
"You are analyzing chart "
"configuration data, not "
"an image. Be concise."
)
),
HumanMessage(
content=chart_summary_prompt
),
]
try:
response = (
chat_model.invoke(
analysis_messages
)
)
analysis_text = (
response.content
)
except Exception:
analysis_text = (
"Chart generated successfully."
)
combined_content = (
f"{image_html}"
f"
{analysis_text}"
)
messages3.append(
HumanMessage(
content=(
"The user requested this chart:\n"
f"{chart_description}"
)
)
)
messages3.append(
AIMessage(
content=(
"A chart was generated with "
"the following Chart.js configuration:\n\n"
f"{json.dumps(chart_config, indent=2)}\n\n"
"Chart URL:\n"
f"{chart_url}"
)
)
)
messages3 = messages3[-6:]
except Exception as e:
combined_content = (
"Chart configuration was generated, "
"but QuickChart could not be reached.\n\n"
f"Error: {str(e)}"
)
else:
prompt = HumanMessage(
content=message
)
messages3.append(
prompt
)
if len(messages3) >= 6:
messages3 = messages3[-6:]
try:
response = chat_model.invoke(
messages3
)
except Exception as e:
print(
"CHART TAB CHAT ERROR:",
repr(e)
)
raise gr.Error(
"Error occurred during response"
) from e
messages3.append(
AIMessage(
content=response.content
)
)
combined_content = (
response.content
)
history.append({
"role": "assistant",
"content": ""
})
for character in combined_content:
history[-1]["content"] += character
time.sleep(
0.0025
)
yield history, messages3
def extract_url_from_text(text):
if not text:
return None
if isinstance(
text,
list
):
parts = []
for item in text:
if isinstance(
item,
dict
):
parts.append(
str(
item.get(
"text",
""
)
)
)
else:
parts.append(
str(item)
)
text = " ".join(parts)
elif isinstance(
text,
dict
):
text = str(
text.get(
"text",
""
)
)
else:
text = str(text)
text = text.replace(
"\\n",
" "
)
text = text.replace(
"\\",
""
)
markdown_matches = re.findall(
r"\]\(\s*(https?://[^)\s]+)",
text,
flags=re.IGNORECASE
)
if markdown_matches:
return markdown_matches[0].rstrip(
".,!?;:)]}"
)
url_matches = re.findall(
r"https?://[^\s<>\"']+",
text,
flags=re.IGNORECASE
)
if url_matches:
return url_matches[0].rstrip(
".,!?;:)]}"
)
return None
def extract_user_query_from_link_message(
content,
link
):
if isinstance(
content,
list
):
parts = []
for item in content:
if isinstance(
item,
dict
):
parts.append(
str(
item.get(
"text",
""
)
)
)
else:
parts.append(
str(item)
)
content = " ".join(parts)
elif isinstance(
content,
dict
):
content = str(
content.get(
"text",
""
)
)
else:
content = str(content)
content = content.replace(
"\\n",
"\n"
)
content = content.replace(
"\\",
""
)
content = content.strip()
question_match = re.search(
r"QUESTION\s*:\s*(.*)$",
content,
flags=re.IGNORECASE | re.DOTALL
)
if question_match:
return question_match.group(
1
).strip()
if link:
question = content.replace(
link,
""
)
question = re.sub(
r"URL\s*:\s*",
"",
question,
flags=re.IGNORECASE
)
return question.strip()
return content
def Link_Scratch(
history,
messages4
):
combined_message = (
history[-1]["content"]
)
if isinstance(
messages4[-1],
HumanMessage
):
messages4 = messages4[:-2]
link = extract_url_from_text(
combined_message
)
user_message = (
extract_user_query_from_link_message(
combined_message,
link
)
)
print(
"RAW LINK REQUEST:",
repr(combined_message)
)
print(
"LINK INPUT:",
repr(link)
)
print(
"USER QUERY:",
repr(user_message)
)
if not link:
response_message = (
"Please provide a valid URL "
"starting with http:// or https://"
)
else:
result = read_data(
link
)
print(
"LINK READ RESULT TYPE:",
type(result)
)
print(
"LINK READ RESULT PREVIEW:",
str(result)[:2000]
)
error_results = [
"Unsupported type or format.",
"Invalid YouTube video URL. "
"Please provide a valid YouTube video link.",
"No English or Arabic transcript "
"was found for this YouTube video.",
"Transcripts are disabled for this "
"YouTube video.",
"The YouTube video is unavailable."
]
if (
isinstance(
result,
str
)
and (
result in error_results
or result.startswith(
"Unable to access webpage"
)
or result.startswith(
"An error occurred while reading"
)
or result.startswith(
"Unable to retrieve the YouTube transcript"
)
or result.startswith(
"No English or Arabic transcript"
)
or result.startswith(
"Transcripts are disabled"
)
or result.startswith(
"The YouTube video is unavailable"
)
)
):
response_message = result
else:
content_data = normalize_text(
result
)
if not content_data:
response_message = (
"The provided link is empty or "
"does not contain any meaningful words."
)
else:
augmented_prompt = f"""
You are an AI designed to help understand
and extract information from provided Link Content.
Based on the user's Query, answer using the provided Link Content.
Query:
{user_message}
Link Content:
{content_data}
Answer the user's query using the Link Content.
"""
message = HumanMessage(
content=augmented_prompt
)
messages4.append(
message
)
messages4 = messages4[-1:]
try:
response = chat_model.invoke(
messages4
)
except Exception as e:
print(
"LINK LLM ERROR:",
repr(e)
)
raise gr.Error(
"Error occurred during response"
) from e
messages4.append(
AIMessage(
content=response.content
)
)
response_message = (
response.content
)
history.append({
"role": "assistant",
"content": ""
})
for character in response_message:
history[-1]["content"] += character
time.sleep(
0.0025
)
yield history, messages4
def insert_line_breaks(
text,
every=8
):
return "\n".join(
text[i:i + every]
for i in range(
0,
len(text),
every
)
)
def display_file_name(
file
):
supported_extensions = [
".csv",
".txt",
".pdf",
".docx",
".pptx",
".xlsx",
".json",
".html",
".xml"
]
file_extension = os.path.splitext(
file.name
)[1]
if (
file_extension.lower()
in supported_extensions
):
file_name = os.path.basename(
file.name
)
file_name_with_breaks = (
insert_line_breaks(
file_name
)
)
icon_url = (
"https://img.icons8.com/"
"ios-filled/50/0000FF/file.png"
)
return (
"