Update app.py
Browse files
app.py
CHANGED
|
@@ -74,7 +74,7 @@ def search_terms_of_service(company_name: str) -> Dict:
|
|
| 74 |
@tool
|
| 75 |
def extract_tos_content(company_name: str) -> Dict:
|
| 76 |
"""
|
| 77 |
-
Search for a company's Terms of Service, access the link, and extract content.
|
| 78 |
|
| 79 |
Args:
|
| 80 |
company_name: Name of the organization/company to search for
|
|
@@ -96,44 +96,55 @@ def extract_tos_content(company_name: str) -> Dict:
|
|
| 96 |
|
| 97 |
print(f"Accessing TOS link: {tos_link}")
|
| 98 |
|
| 99 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
response = requests.get(tos_link)
|
| 101 |
response.raise_for_status()
|
| 102 |
|
| 103 |
# Parse the HTML content
|
| 104 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 105 |
|
| 106 |
-
#
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
tos_element = soup.find('div', id='terms-of-service') or soup.find('div', class_='terms-of-service')
|
| 117 |
-
if tos_element:
|
| 118 |
-
tos_content = tos_element.get_text(strip=True)
|
| 119 |
-
|
| 120 |
-
# If we couldn't find specific elements, try to get the main content
|
| 121 |
-
if not repo_content and not tos_content:
|
| 122 |
-
main_content = soup.find('main') or soup.find('body')
|
| 123 |
if main_content:
|
| 124 |
-
|
| 125 |
-
return {
|
| 126 |
-
"company_name": tos_info["company_name"],
|
| 127 |
-
"tos_link": tos_link,
|
| 128 |
-
"content": main_content_text[:5000], # Limit content length
|
| 129 |
-
"note": "Could not identify specific repository or TOS sections"
|
| 130 |
-
}
|
| 131 |
|
| 132 |
return {
|
| 133 |
"company_name": tos_info["company_name"],
|
| 134 |
"tos_link": tos_link,
|
| 135 |
-
"
|
| 136 |
-
"
|
| 137 |
}
|
| 138 |
|
| 139 |
except requests.exceptions.RequestException as e:
|
|
|
|
| 74 |
@tool
|
| 75 |
def extract_tos_content(company_name: str) -> Dict:
|
| 76 |
"""
|
| 77 |
+
Search for a company's Terms of Service, access the link, and extract raw content from GitLab repository.
|
| 78 |
|
| 79 |
Args:
|
| 80 |
company_name: Name of the organization/company to search for
|
|
|
|
| 96 |
|
| 97 |
print(f"Accessing TOS link: {tos_link}")
|
| 98 |
|
| 99 |
+
# Check if it's a GitLab repository URL
|
| 100 |
+
if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
|
| 101 |
+
# Convert the URL to get the raw content
|
| 102 |
+
# For GitLab, we need to replace '/-/blob/' with '/-/raw/'
|
| 103 |
+
raw_url = tos_link.replace("/-/blob/", "/-/raw/")
|
| 104 |
+
|
| 105 |
+
# Remove any query parameters
|
| 106 |
+
if "?" in raw_url:
|
| 107 |
+
raw_url = raw_url.split("?")[0]
|
| 108 |
+
|
| 109 |
+
print(f"Accessing raw content URL: {raw_url}")
|
| 110 |
+
|
| 111 |
+
# Request the raw content
|
| 112 |
+
response = requests.get(raw_url)
|
| 113 |
+
response.raise_for_status()
|
| 114 |
+
|
| 115 |
+
return {
|
| 116 |
+
"company_name": tos_info["company_name"],
|
| 117 |
+
"tos_link": tos_link,
|
| 118 |
+
"raw_url": raw_url,
|
| 119 |
+
"content": response.text, # Full raw content
|
| 120 |
+
"content_type": "markdown" if raw_url.endswith(".md") else "text"
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
# If it's not a GitLab URL, proceed with regular HTML parsing
|
| 124 |
response = requests.get(tos_link)
|
| 125 |
response.raise_for_status()
|
| 126 |
|
| 127 |
# Parse the HTML content
|
| 128 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 129 |
|
| 130 |
+
# Try to find the content in various ways
|
| 131 |
+
content = ""
|
| 132 |
+
|
| 133 |
+
# Check for pre tags which often contain code or raw text displays
|
| 134 |
+
pre_element = soup.find('pre')
|
| 135 |
+
if pre_element:
|
| 136 |
+
content = pre_element.get_text()
|
| 137 |
+
else:
|
| 138 |
+
# Get the main content if no pre tag
|
| 139 |
+
main_content = soup.find('main') or soup.find('article') or soup.find('body')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
if main_content:
|
| 141 |
+
content = main_content.get_text(strip=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
return {
|
| 144 |
"company_name": tos_info["company_name"],
|
| 145 |
"tos_link": tos_link,
|
| 146 |
+
"content": content,
|
| 147 |
+
"note": "Content extracted from HTML page"
|
| 148 |
}
|
| 149 |
|
| 150 |
except requests.exceptions.RequestException as e:
|