Spaces:
Sleeping
Sleeping
File size: 6,361 Bytes
28611fd 9b5b26a c19d193 58438c3 6aae614 1f81b41 28611fd 8c01ffb ed7b80e 28611fd ed7b80e 28611fd ed7b80e 28611fd 134fbe7 c8592cd 134fbe7 28611fd 8962c06 50093ff 28611fd 8962c06 28611fd ed7b80e 50093ff ed7b80e 28611fd ed7b80e 28611fd ed7b80e 28611fd ed7b80e 28611fd ed7b80e 1f81b41 ed7b80e 1f81b41 a9502de 1f81b41 ed7b80e 1f81b41 861422e 1f81b41 8c01ffb 8fe992b 4069e6a 55260df 8c01ffb 1f81b41 861422e 8fe992b 1f81b41 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, load_tool, tool
import datetime
import requests
import pytz
import yaml
from tools.visit_webpage import VisitWebpageTool
from tools.final_answer import FinalAnswerTool
from Gradio_UI import GradioUI
from typing import List, Dict
from bs4 import BeautifulSoup
@tool
def search_terms_of_service(company_name: str) -> Dict:
"""Search for a company's Terms of Service link
Args:
company_name: Name of the organization/company to search for
Returns:
Dictionary containing company name and link to their Terms of Service
"""
try:
# API URL
# API, ki poskrbi za komunikacijo s podatkovno bazo se poganja lokalno
# Da omogočimo javni dostop do API se uporabi ngrok URL povezava, ki se ob vsakem zagonu spremeni
# Posledično je treba ngrok URL stalno posodabljati
api_base = "https://3aec-176-76-224-7.ngrok-free.app"
# API, ki poskrbi za komunikacijo s podatkovno bazo s
# API request naredimo
url = f"{api_base}/company/{company_name}"
print(f"Request URL: {url}")
response = requests.get(url)
print(f"Response status: {response.status_code}")
print(f"Response body preview: {response.text[:200]}...")
if response.status_code == 404: #HTTP error!
return {"error": f"Company '{company_name}' not found in the database"}
response.raise_for_status()
data = response.json()
# ali obstaja več linkov?
tos_link = None
for link in data['secondary_links']:
if 'terms' in link.lower():
tos_link = link
break
# If no link with "terms" found, fall back to first link
if not tos_link:
tos_link = data["secondary_links"][0]
print(f"Warning: No link containing 'terms' found, using first link: {tos_link}")
# returnamo ime in link, lahko jih več
result = {
"company_name": data["name"],
"tos_link": tos_link,
"all_links": data["secondary_links"] # več linkov
}
return result
except requests.exceptions.RequestException as e:
error_msg = f"API Error: {str(e)}"
if hasattr(e, 'response') and e.response is not None:
error_msg += f" (Status code: {e.response.status_code})"
if hasattr(e.response, 'text'):
error_msg += f" - {e.response.text[:200]}"
print(error_msg) # debug, da prizna, če se zmoti
return {"error": error_msg}
except Exception as e:
error_msg = f"Unexpected error: {str(e)}"
print(error_msg) # debug, da AI prizna, če se zmoti
return {"error": error_msg}
@tool
def extract_tos_content(company_name: str) -> Dict:
"""
Search for a company's Terms of Service, access the link, and extract raw content from GitLab repository.
Args:
company_name: Name of the organization/company to search for
Returns:
Dictionary containing company name, link, and extracted content
"""
tos_info = search_terms_of_service(company_name)
# error
if "error" in tos_info:
return tos_info
try:
# link od zgoraj
tos_link = tos_info["tos_link"]
print(f"Accessing TOS link: {tos_link}")
# Gitlab repo
if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
# spremeni '/-/blob/' z '/-/raw/'
raw_url = tos_link.replace("/-/blob/", "/-/raw/")
if "?" in raw_url:
raw_url = raw_url.split("?")[0]
print(f"Accessing raw content URL: {raw_url}")
# request za raw data iz GItlabaa
response = requests.get(raw_url)
response.raise_for_status()
return {
"company_name": tos_info["company_name"],
"tos_link": tos_link,
"raw_url": raw_url,
"content": response.text, # cel raw data fajl
"content_type": "markdown" if raw_url.endswith(".md") else "text"
}
response = requests.get(tos_link)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
content = ""
pre_element = soup.find('pre')
if pre_element:
content = pre_element.get_text()
else:
main_content = soup.find('main') or soup.find('article') or soup.find('body')
if main_content:
content = main_content.get_text(strip=True)
return {
"company_name": tos_info["company_name"],
"tos_link": tos_link,
"content": content,
"note": "Content extracted from HTML page"
}
except requests.exceptions.RequestException as e:
error_msg = f"Error accessing TOS link: {str(e)}"
if hasattr(e, 'response') and e.response is not None:
error_msg += f" (Status code: {e.response.status_code})"
if hasattr(e.response, 'text'):
error_msg += f" - {e.response.text[:200]}"
print(error_msg)
return {"error": error_msg}
except Exception as e:
error_msg = f"Unexpected error while processing content: {str(e)}"
print(error_msg)
return {"error": error_msg}
final_answer = FinalAnswerTool()
# If the agent does not answer, the model is overloaded, please use another model or the following Hugging Face Endpoint that also contains qwen2.5 coder:
# model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud'
model = HfApiModel(
max_tokens=2096,
temperature=0.5,
model_id='Qwen/Qwen2.5-Coder-32B-Instruct',# it is possible that this model may be overloaded
#model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud',
custom_role_conversions=None,
)
with open("prompts.yaml", 'r') as stream:
prompt_templates = yaml.safe_load(stream)
agent = CodeAgent(
model=model,
tools=[final_answer,search_terms_of_service,extract_tos_content],
max_steps=6,
verbosity_level=1,
grammar=None,
planning_interval=None,
name=None,
description=None,
prompt_templates=prompt_templates
)
GradioUI(agent).launch() |