ncuxomun commited on
Commit
16d5901
·
verified ·
1 Parent(s): bc5b360

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +495 -230
app.py CHANGED
@@ -1,72 +1,73 @@
1
  import os
2
  import gradio as gr
3
  import requests
4
- import inspect
5
  import pandas as pd
6
  from dotenv import load_dotenv
7
- load_dotenv()
8
-
9
- # New imports
10
  from duckduckgo_search import DDGS
11
  import yt_dlp
12
- from PIL import Image
13
- import librosa
14
- import numpy as np
15
- from io import BytesIO
16
  import base64
17
 
18
  from langgraph.graph import StateGraph, END
19
- from langchain_core.runnables import RunnableLambda
20
- from langchain_core.messages import HumanMessage, SystemMessage
21
  from langchain_openai import ChatOpenAI
22
- from typing import TypedDict, Dict, Any
23
  from langchain_core.tools import tool
24
 
25
- # --- New Tools ---
26
-
27
- from langchain_core.tools import tool
28
- from openai import OpenAI
29
- import base64
30
- import requests
31
- import os
32
 
 
 
33
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
34
 
35
- OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
36
-
37
- from duckduckgo_search import DDGS
38
-
39
- # Define types for state schema
40
  class AgentState(TypedDict):
41
  question: str
 
 
42
  answer: str
 
 
 
43
 
44
  @tool
45
- def web_search(query: str, max_results: int = 5) -> str:
46
- """Search the web for information about a given query using DuckDuckGo."""
 
 
 
 
 
 
 
 
 
 
47
  try:
48
  print(f"Searching the web for: {query}")
49
-
50
  results = []
51
  with DDGS() as ddgs:
52
  search_results = list(ddgs.text(query, max_results=max_results))
53
-
54
  for i, result in enumerate(search_results, 1):
55
  title = result.get('title', 'No title')
56
  body = result.get('body', 'No content')
57
  url = result.get('href', 'No URL')
58
- results.append(f"{i}. {title}\n {body}\n URL: {url}\n")
59
-
60
  if not results:
61
  return "No search results found."
62
-
63
- return "\n".join(results)
 
 
64
  except Exception as e:
65
  print(f"Error during web search: {e}")
66
  return f"Error during web search: {str(e)}"
67
 
68
-
69
  def encode_image(image_url):
 
70
  try:
71
  response = requests.get(image_url, stream=True, timeout=10)
72
  response.raise_for_status()
@@ -77,17 +78,33 @@ def encode_image(image_url):
77
 
78
  @tool
79
  def analyze_image_openai_vision(image_url: str, prompt: str) -> str:
80
- """Analyze an image from a given URL using OpenAI Vision and answer a question about it."""
 
 
 
 
 
 
 
 
 
 
81
  if not OPENAI_API_KEY:
82
  return "OpenAI API key not found."
 
83
  base64_image = encode_image(image_url)
84
  if isinstance(base64_image, str) and base64_image.startswith("Error"):
85
  return base64_image
 
86
  client = OpenAI()
87
  try:
88
  response = client.chat.completions.create(
89
  model="gpt-4-vision-preview",
90
  messages=[
 
 
 
 
91
  {
92
  "role": "user",
93
  "content": [
@@ -96,81 +113,16 @@ def analyze_image_openai_vision(image_url: str, prompt: str) -> str:
96
  ],
97
  }
98
  ],
99
- max_tokens=300,
100
- )
101
- return response.choices[0].message.content
102
- except Exception as e:
103
- return f"Error analyzing image with OpenAI Vision: {e}"
104
-
105
- from langchain_core.tools import tool
106
- from openai import OpenAI
107
- import os
108
- import yt_dlp
109
- import requests
110
-
111
- OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
112
-
113
- def get_youtube_transcript(video_url: str) -> str | None:
114
- # (Same implementation as before)
115
- try:
116
- ydl_opts = {
117
- 'writesubtitles': True,
118
- 'writeautomaticsub': True,
119
- 'subtitleslangs': ['en'],
120
- 'skip_download': True,
121
- 'outtmpl': '%(title)s.%(ext)s',
122
- }
123
- with yt_dlp.YoutubeDL(ydl_opts) as ydl:
124
- info_dict = ydl.extract_info(video_url, download=False)
125
- if info_dict and 'subtitles' in info_dict and 'en' in info_dict['subtitles']:
126
- transcript_url = info_dict['subtitles']['en'][0]['url']
127
- response = requests.get(transcript_url, timeout=10)
128
- response.raise_for_status()
129
- transcript_text = "\n".join([line.split(' ', 1)[1] for line in response.text.splitlines() if '-->' not in line and line.strip()])
130
- return transcript_text
131
- elif info_dict and 'automatic_captions' in info_dict and 'en' in info_dict['automatic_captions']:
132
- transcript_url = info_dict['automatic_captions']['en'][0]['url']
133
- response = requests.get(transcript_url, timeout=10)
134
- response.raise_for_status()
135
- transcript_text = "\n".join([line.split(' ', 1)[1] for line in response.text.splitlines() if '-->' not in line and line.strip()])
136
- return transcript_text
137
- else:
138
- return "No English transcript found for this YouTube video."
139
- except Exception as e:
140
- return f"Error fetching YouTube transcript: {e}"
141
-
142
- @tool
143
- def analyze_youtube_video_openai(video_url: str, prompt: str = "Summarize the key points of this video.") -> str:
144
- """Analyze a YouTube video by summarizing its English transcript using OpenAI."""
145
- if not OPENAI_API_KEY:
146
- return "OpenAI API key not found."
147
- transcript = get_youtube_transcript(video_url)
148
- if not transcript:
149
- return "Could not retrieve transcript for the YouTube video."
150
- client = OpenAI()
151
- try:
152
- response = client.chat.completions.create(
153
- model="gpt-4-turbo",
154
- messages=[
155
- {"role": "system", "content": "You are a helpful assistant that analyzes video transcripts."},
156
- {"role": "user", "content": f"Here is the transcript of a video: {transcript[:4000]}. {prompt}"} # Limit transcript length
157
- ],
158
  max_tokens=500,
159
  )
 
160
  return response.choices[0].message.content
161
  except Exception as e:
162
- return f"Error analyzing video transcript with OpenAI: {e}"
163
-
164
- from langchain_core.tools import tool
165
- from openai import OpenAI
166
- import os
167
- import yt_dlp
168
- import requests
169
-
170
- OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
171
 
172
- def get_youtube_transcript(video_url: str) -> str | None:
173
- # (Same implementation as before)
174
  try:
175
  ydl_opts = {
176
  'writesubtitles': True,
@@ -178,187 +130,527 @@ def get_youtube_transcript(video_url: str) -> str | None:
178
  'subtitleslangs': ['en'],
179
  'skip_download': True,
180
  'outtmpl': '%(title)s.%(ext)s',
 
181
  }
 
182
  with yt_dlp.YoutubeDL(ydl_opts) as ydl:
183
  info_dict = ydl.extract_info(video_url, download=False)
 
 
 
 
 
 
184
  if info_dict and 'subtitles' in info_dict and 'en' in info_dict['subtitles']:
185
  transcript_url = info_dict['subtitles']['en'][0]['url']
186
  response = requests.get(transcript_url, timeout=10)
187
  response.raise_for_status()
188
- transcript_text = "\n".join([line.split(' ', 1)[1] for line in response.text.splitlines() if '-->' not in line and line.strip()])
189
- return transcript_text
 
 
 
 
 
 
 
 
 
 
190
  elif info_dict and 'automatic_captions' in info_dict and 'en' in info_dict['automatic_captions']:
191
  transcript_url = info_dict['automatic_captions']['en'][0]['url']
192
  response = requests.get(transcript_url, timeout=10)
193
  response.raise_for_status()
194
- transcript_text = "\n".join([line.split(' ', 1)[1] for line in response.text.splitlines() if '-->' not in line and line.strip()])
195
- return transcript_text
 
 
 
 
 
 
 
196
  else:
197
- return "No English transcript found for this YouTube video."
 
198
  except Exception as e:
 
199
  return f"Error fetching YouTube transcript: {e}"
200
 
201
  @tool
202
  def analyze_youtube_video_openai(video_url: str, prompt: str = "Summarize the key points of this video.") -> str:
203
- """Analyze a YouTube video by summarizing its English transcript using OpenAI."""
 
 
 
 
 
 
 
 
 
 
204
  if not OPENAI_API_KEY:
205
  return "OpenAI API key not found."
 
206
  transcript = get_youtube_transcript(video_url)
207
- if not transcript:
208
  return "Could not retrieve transcript for the YouTube video."
 
209
  client = OpenAI()
210
  try:
211
  response = client.chat.completions.create(
212
  model="gpt-4-turbo",
213
  messages=[
214
- {"role": "system", "content": "You are a helpful assistant that analyzes video transcripts."},
215
- {"role": "user", "content": f"Here is the transcript of a video: {transcript[:4000]}. {prompt}"} # Limit transcript length
 
 
 
 
 
 
216
  ],
217
- max_tokens=500,
218
  )
 
219
  return response.choices[0].message.content
220
  except Exception as e:
 
221
  return f"Error analyzing video transcript with OpenAI: {e}"
222
 
223
- from langchain_core.tools import tool
224
- from openai import OpenAI
225
- import requests
226
- import os
227
-
228
- # OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
229
-
230
- # Python REPL Tool
231
- from langchain_experimental.tools.python.tool import PythonREPLTool
232
- python_repl = PythonREPLTool()
233
-
234
- @tool
235
- def run_python_code(code: str) -> str:
236
- """Execute a Python code snippet and return the output."""
237
- try:
238
- result = python_repl.run(code)
239
- return f"Python code executed successfully. Output: {result}"
240
- except Exception as e:
241
- return f"Error executing Python code: {e}"
242
-
243
  @tool
244
  def transcribe_audio_openai(audio_url: str) -> str:
245
- """Transcribe audio from a given URL using OpenAI Whisper API."""
 
 
 
 
 
 
 
 
 
246
  if not OPENAI_API_KEY:
247
  return "OpenAI API key not found."
 
248
  try:
249
- response = requests.get(audio_url, stream=True, timeout=10)
 
250
  response.raise_for_status()
 
251
  with open("temp_audio.mp3", "wb") as f:
252
  for chunk in response.iter_content(chunk_size=8192):
253
  f.write(chunk)
 
254
  client = OpenAI()
 
255
  with open("temp_audio.mp3", "rb") as audio_file:
256
  transcript = client.audio.transcriptions.create(
257
  model="whisper-1",
258
  file=audio_file
259
  )
 
260
  os.remove("temp_audio.mp3")
 
261
  return transcript.text
262
  except requests.exceptions.RequestException as e:
 
263
  return f"Error accessing audio URL {audio_url}: {e}"
264
  except Exception as e:
 
265
  return f"Error transcribing audio with OpenAI Whisper: {e}"
266
 
267
  @tool
268
  def analyze_audio_transcript_openai(audio_url: str, prompt: str = "Analyze the content of this audio.") -> str:
269
- """Transcribe audio from a URL and then analyze the transcript using OpenAI."""
 
 
 
 
 
 
 
 
 
 
270
  transcript = transcribe_audio_openai(audio_url)
271
  if isinstance(transcript, str) and transcript.startswith("Error"):
272
  return transcript
 
273
  if not transcript:
274
  return "Could not transcribe the audio."
 
275
  client = OpenAI()
276
  try:
277
  response = client.chat.completions.create(
278
  model="gpt-4-turbo",
279
  messages=[
280
- {"role": "system", "content": "You are a helpful assistant that analyzes audio transcripts."},
281
- {"role": "user", "content": f"Here is the transcript of an audio: {transcript[:4000]}. {prompt}"} # Limit transcript length
 
 
 
 
 
 
282
  ],
283
- max_tokens=500,
284
  )
 
285
  return response.choices[0].message.content
286
  except Exception as e:
 
287
  return f"Error analyzing audio transcript with OpenAI: {e}"
288
 
 
 
 
289
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
 
291
- class LangGraphAgent:
292
- def __init__(self):
293
- print("LangGraphAgent initialized.")
294
-
295
- openai_key = os.environ.get("OPENAI_API_KEY")
296
- if not openai_key:
297
- openai_key = os.environ.get("OPEN_AI_KEY")
298
 
299
- if not openai_key:
 
 
 
 
300
  raise ValueError("OpenAI API key not found in environment variables")
301
 
302
- print(f"Loaded OpenAI Key: {openai_key[:3]}..." if openai_key else "No OpenAI Key found")
303
-
304
- self.llm = ChatOpenAI(model="gpt-4-turbo", temperature=0, api_key=openai_key)
305
-
 
306
  # Register all tools
307
- self.tools = [web_search, analyze_image_openai_vision, analyze_youtube_video_openai, analyze_audio_transcript_openai, run_python_code]
 
 
 
 
 
 
 
 
 
 
308
 
 
309
  self.sys_prompt = SystemMessage(
310
- content="""You are a highly capable AI assistant with access to the following tools:
311
- - 'web_search': Use this tool to search the internet for information.
312
- - 'analyze_image_openai_vision': Use this tool to analyze images from URLs and answer questions about their content.
313
- - 'analyze_youtube_video_openai': Use this tool to analyze YouTube videos by summarizing or answering questions based on their transcripts.
314
- - 'analyze_audio_transcript_openai': Use this tool to transcribe audio from URLs and then analyze the transcript.
315
- - 'run_python_code': Use this tool to execute Python code snippets and return the output.
316
-
317
- When I ask you a question, consider if any of these tools can help you find the answer. If so, please use the most appropriate tool(s).
318
-
319
- Your final answer MUST adhere to the following format: <ANSWER> should be a number OR as few words as possible OR a comma-separated list of numbers and/or strings.
320
-
321
- Follow these conditions strictly for your final <ANSWER>:
322
- - If you are asked for a number, do not use commas to write the number and do not include units such as $ or percent signs unless specifically instructed.
323
- - If you are asked for a string, do not use articles (a, an, the), abbreviations (e.g., for cities), and write digits in plain text unless specified otherwise.
324
- - If you are asked for a comma-separated list, apply the above rules to each element in the list, depending on whether it's a number or a string.
325
- - If the question involves Python code, use the 'run_python_code' tool to execute the code and the output as part of your reasoning to arrive at the final <ANSWER>.
326
- """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  )
328
-
329
- def answer_fn(state: AgentState) -> Dict[str, Any]:
 
 
330
  question = state["question"]
331
- print(f"[LangGraphAgent] Received: {question[:50]}...")
332
- messages = [self.sys_prompt, HumanMessage(content=question)]
333
- # You might need to adjust this part to decide when to use which tool.
334
- # A more sophisticated approach would involve a router or a more complex LangGraph workflow.
335
- # For this basic example, the LLM will decide if and when to use the tools
336
- # based on the question and the tool descriptions.
337
- response = self.llm.invoke(messages).content
338
-
339
- final_answer = self._extract_final_answer(response)
340
- return {"question": question, "answer": final_answer}
341
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
342
  workflow = StateGraph(AgentState)
343
- workflow.add_node("answer_question", answer_fn)
344
- workflow.set_entry_point("answer_question")
345
- workflow.add_edge("answer_question", END)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
346
  self.graph = workflow.compile()
 
347
 
348
  def __call__(self, question: str) -> str:
349
- output = self.graph.invoke({"question": question, "answer": ""})
350
- return output["answer"]
351
-
352
- def _extract_final_answer(self, response: str) -> str:
353
- for line in response.strip().splitlines()[::-1]:
354
- if line.strip().lower().startswith("final answer:"):
355
- return line.split(":", 1)[1].strip()
356
- print("Warning: FINAL ANSWER not found, returning full response.")
357
- return response.strip()
358
-
 
 
 
 
 
 
 
 
 
 
 
359
  def run_and_submit_all(profile: gr.OAuthProfile | None):
360
  """
361
- Fetches all questions, runs the BasicAgent/LangGraphAgent on them, submits all answers,
362
  and displays the results.
363
  """
364
  # --- Determine HF Space Runtime URL and Repo URL ---
@@ -375,14 +667,14 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
375
  questions_url = f"{api_url}/questions"
376
  submit_url = f"{api_url}/submit"
377
 
378
- # 1. Instantiate Agent ( modify this part to create your agent)
379
  try:
380
- # agent = BasicAgent()
381
- agent = LangGraphAgent()
382
  except Exception as e:
383
  print(f"Error instantiating agent: {e}")
384
  return f"Error initializing agent: {e}", None
385
- # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
 
386
  agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
387
  print(agent_code)
388
 
@@ -407,7 +699,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
407
  print(f"An unexpected error occurred fetching questions: {e}")
408
  return f"An unexpected error occurred fetching questions: {e}", None
409
 
410
- # 3. Run your Agent
411
  results_log = []
412
  answers_payload = []
413
  print(f"Running agent on {len(questions_data)} questions...")
@@ -418,9 +710,11 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
418
  print(f"Skipping item with missing task_id or question: {item}")
419
  continue
420
  try:
 
421
  submitted_answer = agent(question_text)
422
  answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
423
  results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
 
424
  except Exception as e:
425
  print(f"Error running agent on task {task_id}: {e}")
426
  results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
@@ -447,35 +741,6 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
447
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
448
  f"Message: {result_data.get('message', 'No message received.')}"
449
  )
450
- print("Submission successful.")
451
- results_df = pd.DataFrame(results_log)
452
- return final_status, results_df
453
- except requests.exceptions.HTTPError as e:
454
- error_detail = f"Server responded with status {e.response.status_code}."
455
- try:
456
- error_json = e.response.json()
457
- error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
458
- except requests.exceptions.JSONDecodeError:
459
- error_detail += f" Response: {e.response.text[:500]}"
460
- status_message = f"Submission Failed: {error_detail}"
461
- print(status_message)
462
- results_df = pd.DataFrame(results_log)
463
- return status_message, results_df
464
- except requests.exceptions.Timeout:
465
- status_message = "Submission Failed: The request timed out."
466
- print(status_message)
467
- results_df = pd.DataFrame(results_log)
468
- return status_message, results_df
469
- except requests.exceptions.RequestException as e:
470
- status_message = f"Submission Failed: Network error - {e}"
471
- print(status_message)
472
- results_df = pd.DataFrame(results_log)
473
- return status_message, results_df
474
- except Exception as e:
475
- status_message = f"An unexpected error occurred during submission: {e}"
476
- print(status_message)
477
- results_df = pd.DataFrame(results_log)
478
- return status_message, results_df
479
 
480
 
481
  # --- Build Gradio Interface using Blocks ---
 
1
  import os
2
  import gradio as gr
3
  import requests
 
4
  import pandas as pd
5
  from dotenv import load_dotenv
 
 
 
6
  from duckduckgo_search import DDGS
7
  import yt_dlp
8
+ from openai import OpenAI
 
 
 
9
  import base64
10
 
11
  from langgraph.graph import StateGraph, END
12
+ from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, ToolMessage
 
13
  from langchain_openai import ChatOpenAI
14
+ from typing import TypedDict, Dict, Any, List, Optional, Annotated, Sequence, Union, Literal
15
  from langchain_core.tools import tool
16
 
17
+ load_dotenv()
 
 
 
 
 
 
18
 
19
+ # Environment setup
20
+ OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") or os.environ.get("OPEN_AI_KEY")
21
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
22
 
23
+ # Type definitions for state schema
 
 
 
 
24
  class AgentState(TypedDict):
25
  question: str
26
+ messages: List[Union[HumanMessage, AIMessage, ToolMessage, SystemMessage]]
27
+ steps_remaining: int
28
  answer: str
29
+ intermediate_steps: List[Dict[str, Any]]
30
+
31
+ # --------------------- IMPROVED TOOLS ---------------------
32
 
33
  @tool
34
+ def web_search(query: str, max_results: int = 10) -> str:
35
+ """
36
+ Search the web for information about a given query using DuckDuckGo.
37
+ Provides more comprehensive results than a standard search.
38
+
39
+ Args:
40
+ query: The search query string.
41
+ max_results: Maximum number of search results to return (default: 10)
42
+
43
+ Returns:
44
+ Formatted search results as a string.
45
+ """
46
  try:
47
  print(f"Searching the web for: {query}")
48
+
49
  results = []
50
  with DDGS() as ddgs:
51
  search_results = list(ddgs.text(query, max_results=max_results))
52
+
53
  for i, result in enumerate(search_results, 1):
54
  title = result.get('title', 'No title')
55
  body = result.get('body', 'No content')
56
  url = result.get('href', 'No URL')
57
+ results.append(f"{i}. {title}\n {body}\n URL: {url}\n")
58
+
59
  if not results:
60
  return "No search results found."
61
+
62
+ formatted_results = "\n".join(results)
63
+ print(f"Web search found {len(results)} results")
64
+ return formatted_results
65
  except Exception as e:
66
  print(f"Error during web search: {e}")
67
  return f"Error during web search: {str(e)}"
68
 
 
69
  def encode_image(image_url):
70
+ """Helper function to encode an image from URL to base64."""
71
  try:
72
  response = requests.get(image_url, stream=True, timeout=10)
73
  response.raise_for_status()
 
78
 
79
  @tool
80
  def analyze_image_openai_vision(image_url: str, prompt: str) -> str:
81
+ """
82
+ Analyze an image from a given URL using OpenAI Vision and answer a question about it.
83
+ Provides detailed analysis of visual content.
84
+
85
+ Args:
86
+ image_url: URL of the image to analyze
87
+ prompt: Specific question or instruction about what to analyze in the image
88
+
89
+ Returns:
90
+ Analysis of the image based on the prompt
91
+ """
92
  if not OPENAI_API_KEY:
93
  return "OpenAI API key not found."
94
+
95
  base64_image = encode_image(image_url)
96
  if isinstance(base64_image, str) and base64_image.startswith("Error"):
97
  return base64_image
98
+
99
  client = OpenAI()
100
  try:
101
  response = client.chat.completions.create(
102
  model="gpt-4-vision-preview",
103
  messages=[
104
+ {
105
+ "role": "system",
106
+ "content": "You are a precise image analyzer focusing on exact details and specific information. Extract only what's needed to answer the query."
107
+ },
108
  {
109
  "role": "user",
110
  "content": [
 
113
  ],
114
  }
115
  ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  max_tokens=500,
117
  )
118
+ print(f"Image analysis complete: {len(response.choices[0].message.content)} chars")
119
  return response.choices[0].message.content
120
  except Exception as e:
121
+ print(f"Error analyzing image with OpenAI Vision: {e}")
122
+ return f"Error analyzing image with OpenAI Vision: {e}"
 
 
 
 
 
 
 
123
 
124
+ def get_youtube_transcript(video_url: str) -> str:
125
+ """Extract transcript from a YouTube video with enhanced error handling."""
126
  try:
127
  ydl_opts = {
128
  'writesubtitles': True,
 
130
  'subtitleslangs': ['en'],
131
  'skip_download': True,
132
  'outtmpl': '%(title)s.%(ext)s',
133
+ 'quiet': True,
134
  }
135
+
136
  with yt_dlp.YoutubeDL(ydl_opts) as ydl:
137
  info_dict = ydl.extract_info(video_url, download=False)
138
+ video_title = info_dict.get('title', 'Untitled Video')
139
+ video_duration = info_dict.get('duration', 0)
140
+
141
+ print(f"Video found: '{video_title}' (Duration: {video_duration}s)")
142
+
143
+ # Try official subtitles first
144
  if info_dict and 'subtitles' in info_dict and 'en' in info_dict['subtitles']:
145
  transcript_url = info_dict['subtitles']['en'][0]['url']
146
  response = requests.get(transcript_url, timeout=10)
147
  response.raise_for_status()
148
+
149
+ # More robust transcript parsing
150
+ transcript_lines = []
151
+ for line in response.text.splitlines():
152
+ if '-->' not in line and line.strip():
153
+ parts = line.split(' ', 1)
154
+ if len(parts) > 1:
155
+ transcript_lines.append(parts[1])
156
+
157
+ return f"Video Title: {video_title}\nDuration: {video_duration} seconds\n\nTranscript:\n" + "\n".join(transcript_lines)
158
+
159
+ # Fall back to automatic captions
160
  elif info_dict and 'automatic_captions' in info_dict and 'en' in info_dict['automatic_captions']:
161
  transcript_url = info_dict['automatic_captions']['en'][0]['url']
162
  response = requests.get(transcript_url, timeout=10)
163
  response.raise_for_status()
164
+
165
+ transcript_lines = []
166
+ for line in response.text.splitlines():
167
+ if '-->' not in line and line.strip():
168
+ parts = line.split(' ', 1)
169
+ if len(parts) > 1:
170
+ transcript_lines.append(parts[1])
171
+
172
+ return f"Video Title: {video_title}\nDuration: {video_duration} seconds\n\nTranscript (Auto-generated):\n" + "\n".join(transcript_lines)
173
  else:
174
+ video_info = f"Video Title: {video_title}\nDuration: {video_duration} seconds\n"
175
+ return f"{video_info}\nNo English transcript found for this YouTube video."
176
  except Exception as e:
177
+ print(f"Error fetching YouTube transcript: {e}")
178
  return f"Error fetching YouTube transcript: {e}"
179
 
180
  @tool
181
  def analyze_youtube_video_openai(video_url: str, prompt: str = "Summarize the key points of this video.") -> str:
182
+ """
183
+ Analyze a YouTube video by summarizing its English transcript using OpenAI.
184
+ Provides comprehensive understanding of video content without watching it.
185
+
186
+ Args:
187
+ video_url: URL of the YouTube video to analyze
188
+ prompt: Specific question or instruction about what to analyze in the video
189
+
190
+ Returns:
191
+ Analysis of the video content based on its transcript
192
+ """
193
  if not OPENAI_API_KEY:
194
  return "OpenAI API key not found."
195
+
196
  transcript = get_youtube_transcript(video_url)
197
+ if not transcript or transcript.startswith("Error"):
198
  return "Could not retrieve transcript for the YouTube video."
199
+
200
  client = OpenAI()
201
  try:
202
  response = client.chat.completions.create(
203
  model="gpt-4-turbo",
204
  messages=[
205
+ {
206
+ "role": "system",
207
+ "content": "You are a video content analyzer. Extract precise information from video transcripts. Provide specific details and quantitative data when applicable."
208
+ },
209
+ {
210
+ "role": "user",
211
+ "content": f"Here is the transcript of a video:\n\n{transcript[:7000]}\n\n{prompt}"
212
+ }
213
  ],
214
+ max_tokens=750,
215
  )
216
+ print(f"YouTube video analysis complete: {len(response.choices[0].message.content)} chars")
217
  return response.choices[0].message.content
218
  except Exception as e:
219
+ print(f"Error analyzing video transcript with OpenAI: {e}")
220
  return f"Error analyzing video transcript with OpenAI: {e}"
221
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  @tool
223
  def transcribe_audio_openai(audio_url: str) -> str:
224
+ """
225
+ Transcribe audio from a given URL using OpenAI Whisper API.
226
+ Provides text version of audio content for analysis.
227
+
228
+ Args:
229
+ audio_url: URL of the audio file to transcribe
230
+
231
+ Returns:
232
+ Text transcription of the audio content
233
+ """
234
  if not OPENAI_API_KEY:
235
  return "OpenAI API key not found."
236
+
237
  try:
238
+ print(f"Downloading audio from {audio_url}")
239
+ response = requests.get(audio_url, stream=True, timeout=15)
240
  response.raise_for_status()
241
+
242
  with open("temp_audio.mp3", "wb") as f:
243
  for chunk in response.iter_content(chunk_size=8192):
244
  f.write(chunk)
245
+
246
  client = OpenAI()
247
+ print("Transcribing audio with Whisper API...")
248
  with open("temp_audio.mp3", "rb") as audio_file:
249
  transcript = client.audio.transcriptions.create(
250
  model="whisper-1",
251
  file=audio_file
252
  )
253
+
254
  os.remove("temp_audio.mp3")
255
+ print(f"Audio transcription complete: {len(transcript.text)} chars")
256
  return transcript.text
257
  except requests.exceptions.RequestException as e:
258
+ print(f"Error accessing audio URL {audio_url}: {e}")
259
  return f"Error accessing audio URL {audio_url}: {e}"
260
  except Exception as e:
261
+ print(f"Error transcribing audio with OpenAI Whisper: {e}")
262
  return f"Error transcribing audio with OpenAI Whisper: {e}"
263
 
264
  @tool
265
  def analyze_audio_transcript_openai(audio_url: str, prompt: str = "Analyze the content of this audio.") -> str:
266
+ """
267
+ Transcribe audio from a URL and then analyze the transcript using OpenAI.
268
+ Provides detailed understanding of audio content without listening to it.
269
+
270
+ Args:
271
+ audio_url: URL of the audio file to analyze
272
+ prompt: Specific question or instruction about what to analyze in the audio
273
+
274
+ Returns:
275
+ Analysis of the audio content based on its transcript
276
+ """
277
  transcript = transcribe_audio_openai(audio_url)
278
  if isinstance(transcript, str) and transcript.startswith("Error"):
279
  return transcript
280
+
281
  if not transcript:
282
  return "Could not transcribe the audio."
283
+
284
  client = OpenAI()
285
  try:
286
  response = client.chat.completions.create(
287
  model="gpt-4-turbo",
288
  messages=[
289
+ {
290
+ "role": "system",
291
+ "content": "You are an audio content analyzer. Extract precise information from audio transcripts. Focus on specific details and key points."
292
+ },
293
+ {
294
+ "role": "user",
295
+ "content": f"Here is the transcript of an audio:\n\n{transcript}\n\n{prompt}"
296
+ }
297
  ],
298
+ max_tokens=750,
299
  )
300
+ print(f"Audio analysis complete: {len(response.choices[0].message.content)} chars")
301
  return response.choices[0].message.content
302
  except Exception as e:
303
+ print(f"Error analyzing audio transcript with OpenAI: {e}")
304
  return f"Error analyzing audio transcript with OpenAI: {e}"
305
 
306
+ # Improved Python REPL Tool
307
+ from langchain_experimental.tools.python.tool import PythonREPLTool
308
+ python_repl = PythonREPLTool()
309
 
310
+ @tool
311
+ def run_python_code(code: str) -> str:
312
+ """
313
+ Execute a Python code snippet and return the output.
314
+ Useful for calculations, data analysis, or any computational tasks.
315
+
316
+ Args:
317
+ code: Python code to execute
318
+
319
+ Returns:
320
+ Output of the executed code or error message
321
+ """
322
+ print("Executing Python code...")
323
+ try:
324
+ # Execute the code in a sandbox environment
325
+ result = python_repl.run(code)
326
+ print(f"Python execution successful: {len(str(result))} chars")
327
+ return f"{result}"
328
+ except Exception as e:
329
+ print(f"Error executing Python code: {e}")
330
+ return f"Error executing Python code: {e}"
331
 
332
+ # --------------------- IMPROVED AGENT ---------------------
 
 
 
 
 
 
333
 
334
+ class ImprovedLangGraphAgent:
335
+ def __init__(self):
336
+ print("ImprovedLangGraphAgent initializing...")
337
+
338
+ if not OPENAI_API_KEY:
339
  raise ValueError("OpenAI API key not found in environment variables")
340
 
341
+ print(f"Loaded OpenAI Key: {OPENAI_API_KEY[:3]}..." if OPENAI_API_KEY else "No OpenAI Key found")
342
+
343
+ # Use gpt-4-turbo for better reasoning capabilities
344
+ self.llm = ChatOpenAI(model="gpt-4-turbo", temperature=0, api_key=OPENAI_API_KEY)
345
+
346
  # Register all tools
347
+ self.tools = [
348
+ web_search,
349
+ analyze_image_openai_vision,
350
+ analyze_youtube_video_openai,
351
+ analyze_audio_transcript_openai,
352
+ run_python_code
353
+ ]
354
+
355
+ self.tool_map = {
356
+ tool.name: tool for tool in self.tools
357
+ }
358
 
359
+ # Improved system prompt with clearer instructions and emphasis on reasoning
360
  self.sys_prompt = SystemMessage(
361
+ content="""You are a highly capable AI assistant equipped with tools to solve complex problems. Your goal is to determine the correct answer to the given question, using a step-by-step reasoning approach.
362
+
363
+ AVAILABLE TOOLS:
364
+ - 'web_search': Search the internet for up-to-date information.
365
+ - 'analyze_image_openai_vision': Analyze images from URLs to extract visual information.
366
+ - 'analyze_youtube_video_openai': Analyze YouTube videos by examining their transcripts.
367
+ - 'analyze_audio_transcript_openai': Transcribe and analyze audio from URLs.
368
+ - 'run_python_code': Execute Python code for calculations, data analysis, or visualization.
369
+
370
+ REASONING PROCESS:
371
+ 1. First, understand the question thoroughly.
372
+ 2. Determine which tools would help gather the necessary information.
373
+ 3. Execute each step logically, using tools when needed.
374
+ 4. After collecting information, analyze it critically.
375
+ 5. Draw conclusions based on evidence collected.
376
+ 6. Formulate a precise, concise answer following the format requirements.
377
+
378
+ FINAL ANSWER FORMAT:
379
+ Your final answer MUST be provided in this format:
380
+ <ANSWER>your concise answer here</ANSWER>
381
+
382
+ ANSWER REQUIREMENTS:
383
+ - For numbers: No commas, no units (like $ or %) unless explicitly requested
384
+ - For strings: No articles (a, an, the), no abbreviations for places
385
+ - For lists: Comma-separated values following above rules
386
+
387
+ IMPORTANT:
388
+ - Ensure accuracy above all else
389
+ - Be extremely precise and concise in your final answer
390
+ - For coding questions, execute the code and use the output to determine the answer
391
+ - Never include explanations in your final <ANSWER> tag
392
+ """
393
  )
394
+
395
+ # Define the various components of our graph
396
+ def setup_agent(state: AgentState) -> Dict[str, Any]:
397
+ """Initialize the agent state with the question."""
398
  question = state["question"]
399
+ messages = [
400
+ self.sys_prompt,
401
+ HumanMessage(content=question)
402
+ ]
403
+ print(f"[Agent] Setup with question: {question[:50]}...")
404
+ return {
405
+ "question": question,
406
+ "messages": messages,
407
+ "steps_remaining": 5, # Allow up to 5 reasoning steps
408
+ "answer": "",
409
+ "intermediate_steps": []
410
+ }
411
+
412
+ def route_agent(state: AgentState) -> Literal["process_agent", "finalize_answer"]:
413
+ """Decide whether to continue reasoning or finalize the answer."""
414
+ if state["steps_remaining"] <= 0:
415
+ print("[Agent] Max steps reached, finalizing...")
416
+ return "finalize_answer"
417
+
418
+ # Check if an answer has already been determined
419
+ last_message = state["messages"][-1]
420
+ if isinstance(last_message, AIMessage) and "<ANSWER>" in last_message.content:
421
+ print("[Agent] Answer found, finalizing...")
422
+ return "finalize_answer"
423
+
424
+ print(f"[Agent] Continuing reasoning, {state['steps_remaining']} steps remaining")
425
+ return "process_agent"
426
+
427
+ def process_agent(state: AgentState) -> Dict[str, Any]:
428
+ """Process the current state and decide the next action."""
429
+ messages = state["messages"]
430
+
431
+ # Use the LLM to decide the next step
432
+ response = self.llm.invoke(messages)
433
+ print(f"[Agent] LLM response: {response.content[:75]}...")
434
+
435
+ # Check if the response contains a tool call
436
+ if "I need to use" in response.content.lower() or "let me use" in response.content.lower():
437
+ # Look for tool names in the response
438
+ for tool_name, tool_fn in self.tool_map.items():
439
+ if tool_name.lower() in response.content.lower():
440
+ print(f"[Agent] Using tool: {tool_name}")
441
+ try:
442
+ # Extract arguments from the response
443
+ # This is a simple implementation - could be more sophisticated
444
+ tool_signature = f"{tool_name}("
445
+ start_idx = response.content.lower().find(tool_name.lower())
446
+ if start_idx == -1:
447
+ continue
448
+
449
+ # Find the opening and closing parentheses
450
+ open_paren = response.content.find("(", start_idx)
451
+ if open_paren == -1:
452
+ continue
453
+
454
+ # Extract everything between the first parenthesis and the last one
455
+ paren_count = 1
456
+ close_paren = -1
457
+ for i in range(open_paren + 1, len(response.content)):
458
+ if response.content[i] == "(":
459
+ paren_count += 1
460
+ elif response.content[i] == ")":
461
+ paren_count -= 1
462
+ if paren_count == 0:
463
+ close_paren = i
464
+ break
465
+
466
+ if close_paren == -1:
467
+ continue
468
+
469
+ args_str = response.content[open_paren + 1:close_paren]
470
+
471
+ # Simple parsing of arguments
472
+ # This is a basic implementation - a more robust parser would be better
473
+ args = []
474
+ kwargs = {}
475
+
476
+ # Split by comma, but respecting quotes
477
+ in_quotes = False
478
+ quote_char = None
479
+ current_arg = ""
480
+ for char in args_str:
481
+ if char in ['"', "'"]:
482
+ if not in_quotes:
483
+ in_quotes = True
484
+ quote_char = char
485
+ elif char == quote_char:
486
+ in_quotes = False
487
+ quote_char = None
488
+ current_arg += char
489
+ elif char == ',' and not in_quotes:
490
+ args.append(current_arg.strip())
491
+ current_arg = ""
492
+ else:
493
+ current_arg += char
494
+
495
+ if current_arg:
496
+ args.append(current_arg.strip())
497
+
498
+ # Convert string arguments to actual values
499
+ parsed_args = []
500
+ for arg in args:
501
+ arg = arg.strip()
502
+ if arg.startswith('"') and arg.endswith('"'):
503
+ parsed_args.append(arg[1:-1])
504
+ elif arg.startswith("'") and arg.endswith("'"):
505
+ parsed_args.append(arg[1:-1])
506
+ else:
507
+ # Try to parse as other types
508
+ try:
509
+ parsed_args.append(eval(arg))
510
+ except:
511
+ parsed_args.append(arg)
512
+
513
+ # Execute the tool
514
+ tool_result = tool_fn(*parsed_args)
515
+ print(f"[Agent] Tool result: {tool_result[:75]}...")
516
+
517
+ # Update the state with the tool result
518
+ messages.append(AIMessage(content=response.content))
519
+ messages.append(ToolMessage(content=tool_result, tool_call_id=tool_name))
520
+
521
+ # Record the intermediate step
522
+ state["intermediate_steps"].append({
523
+ "tool": tool_name,
524
+ "args": parsed_args,
525
+ "result": tool_result
526
+ })
527
+
528
+ # Decrement steps remaining
529
+ state["steps_remaining"] -= 1
530
+
531
+ return {
532
+ "question": state["question"],
533
+ "messages": messages,
534
+ "steps_remaining": state["steps_remaining"],
535
+ "answer": state["answer"],
536
+ "intermediate_steps": state["intermediate_steps"]
537
+ }
538
+ except Exception as e:
539
+ print(f"[Agent] Error executing tool {tool_name}: {e}")
540
+ messages.append(AIMessage(content=response.content))
541
+ messages.append(ToolMessage(content=f"Error executing tool {tool_name}: {e}", tool_call_id=tool_name))
542
+ state["steps_remaining"] -= 1
543
+ return state
544
+
545
+ # If no tool call identified, add the response and continue
546
+ messages.append(AIMessage(content=response.content))
547
+ state["steps_remaining"] -= 1
548
+
549
+ return {
550
+ "question": state["question"],
551
+ "messages": messages,
552
+ "steps_remaining": state["steps_remaining"],
553
+ "answer": state["answer"],
554
+ "intermediate_steps": state["intermediate_steps"]
555
+ }
556
+
557
+ def finalize_answer(state: AgentState) -> Dict[str, Any]:
558
+ """Extract the final answer from the agent's reasoning."""
559
+ messages = state["messages"]
560
+ answer = ""
561
+
562
+ # Check if the answer is already in the messages
563
+ for message in reversed(messages):
564
+ if not isinstance(message, HumanMessage) and "<ANSWER>" in message.content:
565
+ # Extract the answer from between <ANSWER> tags
566
+ start_idx = message.content.find("<ANSWER>") + len("<ANSWER>")
567
+ end_idx = message.content.find("</ANSWER>", start_idx)
568
+ if end_idx > start_idx:
569
+ answer = message.content[start_idx:end_idx].strip()
570
+ break
571
+
572
+ # If no answer found, generate one final answer
573
+ if not answer:
574
+ print("[Agent] No answer found, generating final answer...")
575
+ messages.append(HumanMessage(content="Based on all the information we've gathered, what is the final answer? Remember to format it as <ANSWER>your answer here</ANSWER>"))
576
+
577
+ final_response = self.llm.invoke(messages)
578
+ messages.append(final_response)
579
+
580
+ # Extract the answer
581
+ content = final_response.content
582
+ if "<ANSWER>" in content:
583
+ start_idx = content.find("<ANSWER>") + len("<ANSWER>")
584
+ end_idx = content.find("</ANSWER>", start_idx)
585
+ if end_idx > start_idx:
586
+ answer = content[start_idx:end_idx].strip()
587
+ else:
588
+ print("[Agent] Warning: Final response didn't contain <ANSWER> tags")
589
+ answer = content.strip()
590
+
591
+ print(f"[Agent] Final answer: {answer}")
592
+ return {
593
+ "question": state["question"],
594
+ "messages": messages,
595
+ "steps_remaining": state["steps_remaining"],
596
+ "answer": answer,
597
+ "intermediate_steps": state["intermediate_steps"]
598
+ }
599
+
600
+ # Build the graph
601
  workflow = StateGraph(AgentState)
602
+ workflow.add_node("setup", setup_agent)
603
+ workflow.add_node("process_agent", process_agent)
604
+ workflow.add_node("finalize_answer", finalize_answer)
605
+
606
+ # Add edges
607
+ workflow.set_entry_point("setup")
608
+ workflow.add_conditional_edges(
609
+ "setup",
610
+ route_agent,
611
+ {
612
+ "process_agent": "process_agent",
613
+ "finalize_answer": "finalize_answer"
614
+ }
615
+ )
616
+ workflow.add_conditional_edges(
617
+ "process_agent",
618
+ route_agent,
619
+ {
620
+ "process_agent": "process_agent",
621
+ "finalize_answer": "finalize_answer"
622
+ }
623
+ )
624
+ workflow.add_edge("finalize_answer", END)
625
+
626
  self.graph = workflow.compile()
627
+ print("ImprovedLangGraphAgent initialized successfully.")
628
 
629
  def __call__(self, question: str) -> str:
630
+ """Process a question and return the answer."""
631
+ print(f"[ImprovedLangGraphAgent] Processing question: {question[:75]}...")
632
+ output = self.graph.invoke({
633
+ "question": question,
634
+ "messages": [],
635
+ "steps_remaining": 5,
636
+ "answer": "",
637
+ "intermediate_steps": []
638
+ })
639
+
640
+ answer = output["answer"].strip()
641
+ if not answer:
642
+ print("[ImprovedLangGraphAgent] Warning: Empty answer, falling back to last message content")
643
+ # If answer is empty, try to extract it from the last message
644
+ last_message = output["messages"][-1]
645
+ if isinstance(last_message, AIMessage):
646
+ answer = last_message.content.strip()
647
+
648
+ return answer
649
+
650
+ # Update the run_and_submit_all function to use our improved agent
651
  def run_and_submit_all(profile: gr.OAuthProfile | None):
652
  """
653
+ Fetches all questions, runs the ImprovedLangGraphAgent on them, submits all answers,
654
  and displays the results.
655
  """
656
  # --- Determine HF Space Runtime URL and Repo URL ---
 
667
  questions_url = f"{api_url}/questions"
668
  submit_url = f"{api_url}/submit"
669
 
670
+ # 1. Instantiate Agent
671
  try:
672
+ agent = ImprovedLangGraphAgent()
 
673
  except Exception as e:
674
  print(f"Error instantiating agent: {e}")
675
  return f"Error initializing agent: {e}", None
676
+
677
+ # In the case of an app running as a hugging Face space, this link points toward your codebase
678
  agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
679
  print(agent_code)
680
 
 
699
  print(f"An unexpected error occurred fetching questions: {e}")
700
  return f"An unexpected error occurred fetching questions: {e}", None
701
 
702
+ # 3. Run the Agent
703
  results_log = []
704
  answers_payload = []
705
  print(f"Running agent on {len(questions_data)} questions...")
 
710
  print(f"Skipping item with missing task_id or question: {item}")
711
  continue
712
  try:
713
+ print(f"\n--- Processing Task ID: {task_id} ---")
714
  submitted_answer = agent(question_text)
715
  answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
716
  results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
717
+ print(f"--- Task ID: {task_id} completed ---\n")
718
  except Exception as e:
719
  print(f"Error running agent on task {task_id}: {e}")
720
  results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
 
741
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
742
  f"Message: {result_data.get('message', 'No message received.')}"
743
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
744
 
745
 
746
  # --- Build Gradio Interface using Blocks ---