mogg7 commited on
Commit
48f7c0b
·
verified ·
1 Parent(s): fbcb68f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +27 -44
app.py CHANGED
@@ -1,35 +1,16 @@
1
  from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, load_tool, tool
 
2
  import requests
 
3
  import yaml
4
- from typing import List, Dict, Union, TypedDict
5
  from tools.final_answer import FinalAnswerTool
6
  from Gradio_UI import GradioUI
 
7
  from bs4 import BeautifulSoup
8
 
9
- # -----------------------------
10
- # TypedDicts for proper typing
11
- # -----------------------------
12
-
13
- class TOSResult(TypedDict):
14
- company_name: str
15
- tos_link: str
16
- all_links: List[str]
17
-
18
- class TOSContent(TypedDict, total=False):
19
- company_name: str
20
- tos_link: str
21
- content: str
22
- raw_url: str
23
- content_type: str
24
- note: str
25
-
26
-
27
- # -----------------------------
28
- # Tool: Search Terms of Service
29
- # -----------------------------
30
 
31
  @tool
32
- def search_terms_of_service(company_name: str) -> Union[TOSResult, Dict[str, str]]:
33
  """
34
  Search for a company's Terms of Service link.
35
 
@@ -37,14 +18,14 @@ def search_terms_of_service(company_name: str) -> Union[TOSResult, Dict[str, str
37
  company_name (str): The name of the organization or company to search for.
38
 
39
  Returns:
40
- Union[TOSResult, Dict[str, str]]: A dictionary containing the TOS links or an error message.
41
  """
42
 
43
  try:
44
  api_base = "https://05bf-176-76-227-247.ngrok-free.app"
45
  url = f"{api_base}/company/{company_name}"
46
-
47
  print(f"Request URL: {url}")
 
48
  response = requests.get(url)
49
  print(f"Response status: {response.status_code}")
50
  print(f"Response body preview: {response.text[:200]}...")
@@ -58,14 +39,20 @@ def search_terms_of_service(company_name: str) -> Union[TOSResult, Dict[str, str
58
  if not data.get('secondary_links'):
59
  return {"error": f"No Terms of Service links found for {company_name}"}
60
 
61
- return {
62
  "company_name": data["name"],
63
  "tos_link": data["secondary_links"][0],
64
  "all_links": data["secondary_links"]
65
  }
66
 
 
 
67
  except requests.exceptions.RequestException as e:
68
  error_msg = f"API Error: {str(e)}"
 
 
 
 
69
  print(error_msg)
70
  return {"error": error_msg}
71
 
@@ -75,12 +62,8 @@ def search_terms_of_service(company_name: str) -> Union[TOSResult, Dict[str, str
75
  return {"error": error_msg}
76
 
77
 
78
- # -----------------------------
79
- # Tool: Extract ToS Content
80
- # -----------------------------
81
-
82
  @tool
83
- def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
84
  """
85
  Fetch and extract the Terms of Service content for a company.
86
 
@@ -88,10 +71,9 @@ def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
88
  company_name (str): The name of the organization or company to search for.
89
 
90
  Returns:
91
- Union[TOSContent, Dict[str, str]]: Extracted content and metadata or an error message.
92
  """
93
 
94
-
95
  tos_info = search_terms_of_service(company_name)
96
 
97
  if "error" in tos_info:
@@ -101,11 +83,12 @@ def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
101
  tos_link = tos_info["tos_link"]
102
  print(f"Accessing TOS link: {tos_link}")
103
 
104
- # GitLab raw conversion
105
  if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
106
- raw_url = tos_link.replace("/-/blob/", "/-/raw/").split("?")[0]
107
- print(f"Accessing raw content URL: {raw_url}")
 
108
 
 
109
  response = requests.get(raw_url)
110
  response.raise_for_status()
111
 
@@ -117,7 +100,6 @@ def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
117
  "content_type": "markdown" if raw_url.endswith(".md") else "text"
118
  }
119
 
120
- # Fallback to HTML scraping
121
  response = requests.get(tos_link)
122
  response.raise_for_status()
123
 
@@ -141,6 +123,10 @@ def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
141
 
142
  except requests.exceptions.RequestException as e:
143
  error_msg = f"Error accessing TOS link: {str(e)}"
 
 
 
 
144
  print(error_msg)
145
  return {"error": error_msg}
146
 
@@ -150,12 +136,10 @@ def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
150
  return {"error": error_msg}
151
 
152
 
153
- # -----------------------------
154
- # Final Answer Tool & Agent
155
- # -----------------------------
156
-
157
  final_answer = FinalAnswerTool()
158
 
 
159
  model = HfApiModel(
160
  max_tokens=2096,
161
  temperature=0.3,
@@ -163,9 +147,11 @@ model = HfApiModel(
163
  custom_role_conversions=None,
164
  )
165
 
 
166
  with open("prompts.yaml", 'r') as stream:
167
  prompt_templates = yaml.safe_load(stream)
168
 
 
169
  agent = CodeAgent(
170
  model=model,
171
  tools=[final_answer, search_terms_of_service, extract_tos_content],
@@ -178,8 +164,5 @@ agent = CodeAgent(
178
  prompt_templates=prompt_templates
179
  )
180
 
181
- # -----------------------------
182
  # Launch Gradio UI
183
- # -----------------------------
184
-
185
  GradioUI(agent).launch()
 
1
  from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, load_tool, tool
2
+ import datetime
3
  import requests
4
+ import pytz
5
  import yaml
 
6
  from tools.final_answer import FinalAnswerTool
7
  from Gradio_UI import GradioUI
8
+ from typing import List, Dict, Any
9
  from bs4 import BeautifulSoup
10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
  @tool
13
+ def search_terms_of_service(company_name: str) -> Dict[str, Any]:
14
  """
15
  Search for a company's Terms of Service link.
16
 
 
18
  company_name (str): The name of the organization or company to search for.
19
 
20
  Returns:
21
+ Dict[str, Any]: A dictionary containing company name and TOS link(s), or an error message.
22
  """
23
 
24
  try:
25
  api_base = "https://05bf-176-76-227-247.ngrok-free.app"
26
  url = f"{api_base}/company/{company_name}"
 
27
  print(f"Request URL: {url}")
28
+
29
  response = requests.get(url)
30
  print(f"Response status: {response.status_code}")
31
  print(f"Response body preview: {response.text[:200]}...")
 
39
  if not data.get('secondary_links'):
40
  return {"error": f"No Terms of Service links found for {company_name}"}
41
 
42
+ result = {
43
  "company_name": data["name"],
44
  "tos_link": data["secondary_links"][0],
45
  "all_links": data["secondary_links"]
46
  }
47
 
48
+ return result
49
+
50
  except requests.exceptions.RequestException as e:
51
  error_msg = f"API Error: {str(e)}"
52
+ if hasattr(e, 'response') and e.response is not None:
53
+ error_msg += f" (Status code: {e.response.status_code})"
54
+ if hasattr(e.response, 'text'):
55
+ error_msg += f" - {e.response.text[:200]}"
56
  print(error_msg)
57
  return {"error": error_msg}
58
 
 
62
  return {"error": error_msg}
63
 
64
 
 
 
 
 
65
  @tool
66
+ def extract_tos_content(company_name: str) -> Dict[str, Any]:
67
  """
68
  Fetch and extract the Terms of Service content for a company.
69
 
 
71
  company_name (str): The name of the organization or company to search for.
72
 
73
  Returns:
74
+ Dict[str, Any]: Dictionary containing TOS content and metadata, or an error message.
75
  """
76
 
 
77
  tos_info = search_terms_of_service(company_name)
78
 
79
  if "error" in tos_info:
 
83
  tos_link = tos_info["tos_link"]
84
  print(f"Accessing TOS link: {tos_link}")
85
 
 
86
  if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
87
+ raw_url = tos_link.replace("/-/blob/", "/-/raw/")
88
+ if "?" in raw_url:
89
+ raw_url = raw_url.split("?")[0]
90
 
91
+ print(f"Accessing raw content URL: {raw_url}")
92
  response = requests.get(raw_url)
93
  response.raise_for_status()
94
 
 
100
  "content_type": "markdown" if raw_url.endswith(".md") else "text"
101
  }
102
 
 
103
  response = requests.get(tos_link)
104
  response.raise_for_status()
105
 
 
123
 
124
  except requests.exceptions.RequestException as e:
125
  error_msg = f"Error accessing TOS link: {str(e)}"
126
+ if hasattr(e, 'response') and e.response is not None:
127
+ error_msg += f" (Status code: {e.response.status_code})"
128
+ if hasattr(e.response, 'text'):
129
+ error_msg += f" - {e.response.text[:200]}"
130
  print(error_msg)
131
  return {"error": error_msg}
132
 
 
136
  return {"error": error_msg}
137
 
138
 
139
+ # Tools
 
 
 
140
  final_answer = FinalAnswerTool()
141
 
142
+ # Hugging Face model
143
  model = HfApiModel(
144
  max_tokens=2096,
145
  temperature=0.3,
 
147
  custom_role_conversions=None,
148
  )
149
 
150
+ # Load prompt templates
151
  with open("prompts.yaml", 'r') as stream:
152
  prompt_templates = yaml.safe_load(stream)
153
 
154
+ # Agent configuration
155
  agent = CodeAgent(
156
  model=model,
157
  tools=[final_answer, search_terms_of_service, extract_tos_content],
 
164
  prompt_templates=prompt_templates
165
  )
166
 
 
167
  # Launch Gradio UI
 
 
168
  GradioUI(agent).launch()