Update tools/visit_webpage.py

tools/visit_webpage.py

@@ -11,6 +11,10 @@ class VisitWebpageTool(Tool):
     inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
     output_type = "string"
 
+    HEADERS = {
+        "User-Agent": "Mozilla/5.0 (compatible; ResearchAgent/1.0; +https://huggingface.co/spaces/Realistvik0/First_agent_template)"
+    }
+
     def forward(self, url: str) -> str:
         try:
             import requests
@@ -23,8 +27,8 @@ class VisitWebpageTool(Tool):
                 "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
             ) from e
         try:
-            # Send a GET request to the URL with a 20-second timeout
-            response = requests.get(url, timeout=20)
+            # Send a GET request to the URL with a real User-Agent and a 20-second timeout
+            response = requests.get(url, headers=self.HEADERS, timeout=20)
             response.raise_for_status()  # Raise an exception for bad status codes
 
             # Convert the HTML content to Markdown
@@ -43,4 +47,4 @@ class VisitWebpageTool(Tool):
             return f"An unexpected error occurred: {str(e)}"
 
     def __init__(self, *args, **kwargs):
-        self.is_initialized = False
+        self.is_initialized = False