File size: 6,361 Bytes
28611fd
9b5b26a
 
 
c19d193
58438c3
6aae614
1f81b41
28611fd
 
 
 
8c01ffb
ed7b80e
28611fd
 
 
ed7b80e
28611fd
 
 
 
ed7b80e
28611fd
 
 
134fbe7
 
 
c8592cd
134fbe7
28611fd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8962c06
 
 
 
 
 
 
 
 
 
 
 
50093ff
28611fd
8962c06
28611fd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ed7b80e
50093ff
ed7b80e
28611fd
 
 
 
ed7b80e
28611fd
 
 
 
ed7b80e
28611fd
 
 
 
 
 
 
ed7b80e
28611fd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ed7b80e
 
 
 
1f81b41
 
 
ed7b80e
1f81b41
 
a9502de
 
1f81b41
ed7b80e
 
1f81b41
861422e
 
1f81b41
8c01ffb
8fe992b
4069e6a
55260df
8c01ffb
 
 
1f81b41
 
861422e
8fe992b
 
1f81b41
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, load_tool, tool
import datetime
import requests
import pytz
import yaml
from tools.visit_webpage import VisitWebpageTool
from tools.final_answer import FinalAnswerTool
from Gradio_UI import GradioUI
from typing import List, Dict
from bs4 import BeautifulSoup



@tool
def search_terms_of_service(company_name: str) -> Dict:
    """Search for a company's Terms of Service link

    Args:
        company_name: Name of the organization/company to search for

    Returns:
        Dictionary containing company name and link to their Terms of Service
    """

    try:
        # API URL
        # API, ki poskrbi za komunikacijo s podatkovno bazo se poganja lokalno
        # Da omogočimo javni dostop do API se uporabi ngrok URL povezava, ki se ob vsakem zagonu spremeni
        # Posledično je treba ngrok URL stalno posodabljati
        api_base = "https://3aec-176-76-224-7.ngrok-free.app"  
        # API, ki poskrbi za komunikacijo s podatkovno bazo s

        # API request naredimo
        url = f"{api_base}/company/{company_name}"

        print(f"Request URL: {url}")

        response = requests.get(url)

        print(f"Response status: {response.status_code}")
        print(f"Response body preview: {response.text[:200]}...")

        if response.status_code == 404: #HTTP error!
            return {"error": f"Company '{company_name}' not found in the database"}

        response.raise_for_status()

        data = response.json()

        # ali obstaja več linkov?
        tos_link = None
        for link in data['secondary_links']:
            if 'terms' in link.lower():
                tos_link = link
                break
        
        # If no link with "terms" found, fall back to first link
        if not tos_link:
            tos_link = data["secondary_links"][0]
            print(f"Warning: No link containing 'terms' found, using first link: {tos_link}")
        
        # returnamo ime in link, lahko jih več
        result = {
            "company_name": data["name"],
            "tos_link": tos_link,
            "all_links": data["secondary_links"]  # več linkov
        }
        return result

    except requests.exceptions.RequestException as e:
        error_msg = f"API Error: {str(e)}"
        if hasattr(e, 'response') and e.response is not None:
            error_msg += f" (Status code: {e.response.status_code})"
            if hasattr(e.response, 'text'):
                error_msg += f" - {e.response.text[:200]}"
        print(error_msg)  # debug, da prizna, če se zmoti
        return {"error": error_msg}

    except Exception as e:
        error_msg = f"Unexpected error: {str(e)}"
        print(error_msg)  # debug, da AI prizna, če se zmoti
        return {"error": error_msg}

        
@tool
def extract_tos_content(company_name: str) -> Dict:
    """
    Search for a company's Terms of Service, access the link, and extract raw content from GitLab repository.

    Args:
        company_name: Name of the organization/company to search for

    Returns:
        Dictionary containing company name, link, and extracted content
    """

    tos_info = search_terms_of_service(company_name)

    # error
    if "error" in tos_info:
        return tos_info

    try:
        # link od zgoraj
        tos_link = tos_info["tos_link"]

        print(f"Accessing TOS link: {tos_link}")

        # Gitlab repo
        if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
            # spremeni '/-/blob/' z '/-/raw/'
            raw_url = tos_link.replace("/-/blob/", "/-/raw/")


            if "?" in raw_url:
                raw_url = raw_url.split("?")[0]

            print(f"Accessing raw content URL: {raw_url}")

            # request za raw data iz GItlabaa
            response = requests.get(raw_url)
            response.raise_for_status()

            return {
                "company_name": tos_info["company_name"],
                "tos_link": tos_link,
                "raw_url": raw_url,
                "content": response.text,  # cel raw data fajl
                "content_type": "markdown" if raw_url.endswith(".md") else "text"
            }

        response = requests.get(tos_link)
        response.raise_for_status()

        soup = BeautifulSoup(response.text, 'html.parser')

        content = ""

        pre_element = soup.find('pre')
        if pre_element:
            content = pre_element.get_text()
        else:
            main_content = soup.find('main') or soup.find('article') or soup.find('body')
            if main_content:
                content = main_content.get_text(strip=True)

        return {
            "company_name": tos_info["company_name"],
            "tos_link": tos_link,
            "content": content,
            "note": "Content extracted from HTML page"
        }

    except requests.exceptions.RequestException as e:
        error_msg = f"Error accessing TOS link: {str(e)}"
        if hasattr(e, 'response') and e.response is not None:
            error_msg += f" (Status code: {e.response.status_code})"
            if hasattr(e.response, 'text'):
                error_msg += f" - {e.response.text[:200]}"
        print(error_msg)
        return {"error": error_msg}

    except Exception as e:
        error_msg = f"Unexpected error while processing content: {str(e)}"
        print(error_msg)
        return {"error": error_msg}



final_answer = FinalAnswerTool()

# If the agent does not answer, the model is overloaded, please use another model or the following Hugging Face Endpoint that also contains qwen2.5 coder:
# model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud' 

model = HfApiModel(
max_tokens=2096,
temperature=0.5,
model_id='Qwen/Qwen2.5-Coder-32B-Instruct',# it is possible that this model may be overloaded
#model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud',
custom_role_conversions=None,
)


with open("prompts.yaml", 'r') as stream:
    prompt_templates = yaml.safe_load(stream)
    
agent = CodeAgent(
    model=model,
    tools=[final_answer,search_terms_of_service,extract_tos_content], 
    max_steps=6,
    verbosity_level=1,
    grammar=None,
    planning_interval=None,
    name=None,
    description=None,
    prompt_templates=prompt_templates
)


GradioUI(agent).launch()