mogg7 commited on
Commit
bdc3020
·
verified ·
1 Parent(s): 9397fc2

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +72 -87
app.py CHANGED
@@ -1,127 +1,112 @@
1
  from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, load_tool, tool
2
- import datetime
3
  import requests
4
- import pytz
5
  import yaml
 
6
  from tools.final_answer import FinalAnswerTool
7
  from Gradio_UI import GradioUI
8
- from typing import List, Dict
9
  from bs4 import BeautifulSoup
10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
  @tool
13
- def search_terms_of_service(company_name: str) -> Dict:
14
- """Search for a company's Terms of Service link
15
-
16
- Args:
17
- company_name: Name of the organization/company to search for
18
-
19
- Returns:
20
- Dictionary containing company name and link to their Terms of Service
21
- """
22
-
23
  try:
24
- # API URL
25
- api_base = "https://05bf-176-76-227-247.ngrok-free.app" # lokalno ga runnamo
26
-
27
- # API request naredimo
28
  url = f"{api_base}/company/{company_name}"
29
-
30
  print(f"Request URL: {url}")
31
-
32
  response = requests.get(url)
33
-
34
  print(f"Response status: {response.status_code}")
35
  print(f"Response body preview: {response.text[:200]}...")
36
-
37
- if response.status_code == 404: #HTTP error!
38
  return {"error": f"Company '{company_name}' not found in the database"}
39
-
40
  response.raise_for_status()
41
-
42
  data = response.json()
43
-
44
- # ali obstaja več linkov?
45
- if not data.get('secondary_links') or len(data['secondary_links']) == 0:
46
  return {"error": f"No Terms of Service links found for {company_name}"}
47
-
48
- # returnamo ime in link, lahko jih več
49
- result = {
50
  "company_name": data["name"],
51
- "tos_link": data["secondary_links"][0], # prvi link
52
- "all_links": data["secondary_links"] # več linkov
53
  }
54
-
55
- return result
56
-
57
  except requests.exceptions.RequestException as e:
58
  error_msg = f"API Error: {str(e)}"
59
- if hasattr(e, 'response') and e.response is not None:
60
- error_msg += f" (Status code: {e.response.status_code})"
61
- if hasattr(e.response, 'text'):
62
- error_msg += f" - {e.response.text[:200]}"
63
- print(error_msg) # debug, da prizna, če se zmoti
64
  return {"error": error_msg}
65
-
66
  except Exception as e:
67
  error_msg = f"Unexpected error: {str(e)}"
68
- print(error_msg) # debug, da AI prizna, če se zmoti
69
  return {"error": error_msg}
70
 
 
 
 
 
 
71
  @tool
72
- def extract_tos_content(company_name: str) -> Dict:
73
  """
74
- Search for a company's Terms of Service, access the link, and extract raw content from GitLab repository.
75
-
76
- Args:
77
- company_name: Name of the organization/company to search for
78
-
79
- Returns:
80
- Dictionary containing company name, link, and extracted content
81
  """
82
-
83
  tos_info = search_terms_of_service(company_name)
84
-
85
- # error
86
  if "error" in tos_info:
87
  return tos_info
88
-
89
  try:
90
- # link od zgoraj
91
  tos_link = tos_info["tos_link"]
92
-
93
  print(f"Accessing TOS link: {tos_link}")
94
-
95
- # Gitlab repo
96
  if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
97
- # spremeni '/-/blob/' z '/-/raw/'
98
- raw_url = tos_link.replace("/-/blob/", "/-/raw/")
99
-
100
-
101
- if "?" in raw_url:
102
- raw_url = raw_url.split("?")[0]
103
-
104
  print(f"Accessing raw content URL: {raw_url}")
105
-
106
- # request za raw data iz GItlabaa
107
  response = requests.get(raw_url)
108
  response.raise_for_status()
109
-
110
  return {
111
  "company_name": tos_info["company_name"],
112
  "tos_link": tos_link,
113
  "raw_url": raw_url,
114
- "content": response.text, # cel raw data fajl
115
  "content_type": "markdown" if raw_url.endswith(".md") else "text"
116
  }
117
-
 
118
  response = requests.get(tos_link)
119
  response.raise_for_status()
120
-
121
  soup = BeautifulSoup(response.text, 'html.parser')
122
-
123
  content = ""
124
-
125
  pre_element = soup.find('pre')
126
  if pre_element:
127
  content = pre_element.get_text()
@@ -129,48 +114,44 @@ def extract_tos_content(company_name: str) -> Dict:
129
  main_content = soup.find('main') or soup.find('article') or soup.find('body')
130
  if main_content:
131
  content = main_content.get_text(strip=True)
132
-
133
  return {
134
  "company_name": tos_info["company_name"],
135
  "tos_link": tos_link,
136
  "content": content,
137
  "note": "Content extracted from HTML page"
138
  }
139
-
140
  except requests.exceptions.RequestException as e:
141
  error_msg = f"Error accessing TOS link: {str(e)}"
142
- if hasattr(e, 'response') and e.response is not None:
143
- error_msg += f" (Status code: {e.response.status_code})"
144
- if hasattr(e.response, 'text'):
145
- error_msg += f" - {e.response.text[:200]}"
146
  print(error_msg)
147
  return {"error": error_msg}
148
-
149
  except Exception as e:
150
  error_msg = f"Unexpected error while processing content: {str(e)}"
151
  print(error_msg)
152
  return {"error": error_msg}
153
 
154
-
 
 
 
155
 
156
  final_answer = FinalAnswerTool()
157
 
158
- #kateri model in kje se povežemo!
159
  model = HfApiModel(
160
  max_tokens=2096,
161
- temperature=0.3,
162
  model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud/',
163
  custom_role_conversions=None,
164
  )
165
 
166
- #prompti za agenta
167
  with open("prompts.yaml", 'r') as stream:
168
  prompt_templates = yaml.safe_load(stream)
169
 
170
- #param za agenta
171
  agent = CodeAgent(
172
  model=model,
173
- tools=[final_answer,search_terms_of_service,extract_tos_content], #search_court_cases, check_citation_status], #orodja
174
  max_steps=6,
175
  verbosity_level=1,
176
  grammar=None,
@@ -180,4 +161,8 @@ agent = CodeAgent(
180
  prompt_templates=prompt_templates
181
  )
182
 
183
- GradioUI(agent).launch() #launch za UI
 
 
 
 
 
1
  from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, load_tool, tool
 
2
  import requests
 
3
  import yaml
4
+ from typing import List, Dict, Union, TypedDict
5
  from tools.final_answer import FinalAnswerTool
6
  from Gradio_UI import GradioUI
 
7
  from bs4 import BeautifulSoup
8
 
9
+ # -----------------------------
10
+ # TypedDicts for proper typing
11
+ # -----------------------------
12
+
13
+ class TOSResult(TypedDict):
14
+ company_name: str
15
+ tos_link: str
16
+ all_links: List[str]
17
+
18
+ class TOSContent(TypedDict, total=False):
19
+ company_name: str
20
+ tos_link: str
21
+ content: str
22
+ raw_url: str
23
+ content_type: str
24
+ note: str
25
+
26
+
27
+ # -----------------------------
28
+ # Tool: Search Terms of Service
29
+ # -----------------------------
30
 
31
  @tool
32
+ def search_terms_of_service(company_name: str) -> Union[TOSResult, Dict[str, str]]:
33
+ """Search for a company's Terms of Service link."""
 
 
 
 
 
 
 
 
34
  try:
35
+ api_base = "https://05bf-176-76-227-247.ngrok-free.app"
 
 
 
36
  url = f"{api_base}/company/{company_name}"
37
+
38
  print(f"Request URL: {url}")
 
39
  response = requests.get(url)
 
40
  print(f"Response status: {response.status_code}")
41
  print(f"Response body preview: {response.text[:200]}...")
42
+
43
+ if response.status_code == 404:
44
  return {"error": f"Company '{company_name}' not found in the database"}
45
+
46
  response.raise_for_status()
 
47
  data = response.json()
48
+
49
+ if not data.get('secondary_links'):
 
50
  return {"error": f"No Terms of Service links found for {company_name}"}
51
+
52
+ return {
 
53
  "company_name": data["name"],
54
+ "tos_link": data["secondary_links"][0],
55
+ "all_links": data["secondary_links"]
56
  }
57
+
 
 
58
  except requests.exceptions.RequestException as e:
59
  error_msg = f"API Error: {str(e)}"
60
+ print(error_msg)
 
 
 
 
61
  return {"error": error_msg}
62
+
63
  except Exception as e:
64
  error_msg = f"Unexpected error: {str(e)}"
65
+ print(error_msg)
66
  return {"error": error_msg}
67
 
68
+
69
+ # -----------------------------
70
+ # Tool: Extract ToS Content
71
+ # -----------------------------
72
+
73
  @tool
74
+ def extract_tos_content(company_name: str) -> Union[TOSContent, Dict[str, str]]:
75
  """
76
+ Fetch and extract Terms of Service content for a given company.
 
 
 
 
 
 
77
  """
 
78
  tos_info = search_terms_of_service(company_name)
79
+
 
80
  if "error" in tos_info:
81
  return tos_info
82
+
83
  try:
 
84
  tos_link = tos_info["tos_link"]
 
85
  print(f"Accessing TOS link: {tos_link}")
86
+
87
+ # GitLab raw conversion
88
  if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
89
+ raw_url = tos_link.replace("/-/blob/", "/-/raw/").split("?")[0]
 
 
 
 
 
 
90
  print(f"Accessing raw content URL: {raw_url}")
91
+
 
92
  response = requests.get(raw_url)
93
  response.raise_for_status()
94
+
95
  return {
96
  "company_name": tos_info["company_name"],
97
  "tos_link": tos_link,
98
  "raw_url": raw_url,
99
+ "content": response.text,
100
  "content_type": "markdown" if raw_url.endswith(".md") else "text"
101
  }
102
+
103
+ # Fallback to HTML scraping
104
  response = requests.get(tos_link)
105
  response.raise_for_status()
106
+
107
  soup = BeautifulSoup(response.text, 'html.parser')
 
108
  content = ""
109
+
110
  pre_element = soup.find('pre')
111
  if pre_element:
112
  content = pre_element.get_text()
 
114
  main_content = soup.find('main') or soup.find('article') or soup.find('body')
115
  if main_content:
116
  content = main_content.get_text(strip=True)
117
+
118
  return {
119
  "company_name": tos_info["company_name"],
120
  "tos_link": tos_link,
121
  "content": content,
122
  "note": "Content extracted from HTML page"
123
  }
124
+
125
  except requests.exceptions.RequestException as e:
126
  error_msg = f"Error accessing TOS link: {str(e)}"
 
 
 
 
127
  print(error_msg)
128
  return {"error": error_msg}
129
+
130
  except Exception as e:
131
  error_msg = f"Unexpected error while processing content: {str(e)}"
132
  print(error_msg)
133
  return {"error": error_msg}
134
 
135
+
136
+ # -----------------------------
137
+ # Final Answer Tool & Agent
138
+ # -----------------------------
139
 
140
  final_answer = FinalAnswerTool()
141
 
 
142
  model = HfApiModel(
143
  max_tokens=2096,
144
+ temperature=0.3,
145
  model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud/',
146
  custom_role_conversions=None,
147
  )
148
 
 
149
  with open("prompts.yaml", 'r') as stream:
150
  prompt_templates = yaml.safe_load(stream)
151
 
 
152
  agent = CodeAgent(
153
  model=model,
154
+ tools=[final_answer, search_terms_of_service, extract_tos_content],
155
  max_steps=6,
156
  verbosity_level=1,
157
  grammar=None,
 
161
  prompt_templates=prompt_templates
162
  )
163
 
164
+ # -----------------------------
165
+ # Launch Gradio UI
166
+ # -----------------------------
167
+
168
+ GradioUI(agent).launch()