Update app.py
Browse files
app.py
CHANGED
|
@@ -83,10 +83,9 @@ def extract_tos_content(company_name: str) -> Dict:
|
|
| 83 |
Dictionary containing company name, link, and extracted content
|
| 84 |
"""
|
| 85 |
|
| 86 |
-
# First use the existing function to get the TOS link
|
| 87 |
tos_info = search_terms_of_service(company_name)
|
| 88 |
|
| 89 |
-
#
|
| 90 |
if "error" in tos_info:
|
| 91 |
return tos_info
|
| 92 |
|
|
@@ -96,13 +95,12 @@ def extract_tos_content(company_name: str) -> Dict:
|
|
| 96 |
|
| 97 |
print(f"Accessing TOS link: {tos_link}")
|
| 98 |
|
| 99 |
-
#
|
| 100 |
if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
|
| 101 |
-
#
|
| 102 |
-
# For GitLab, we need to replace '/-/blob/' with '/-/raw/'
|
| 103 |
raw_url = tos_link.replace("/-/blob/", "/-/raw/")
|
| 104 |
|
| 105 |
-
|
| 106 |
if "?" in raw_url:
|
| 107 |
raw_url = raw_url.split("?")[0]
|
| 108 |
|
|
@@ -120,22 +118,17 @@ def extract_tos_content(company_name: str) -> Dict:
|
|
| 120 |
"content_type": "markdown" if raw_url.endswith(".md") else "text"
|
| 121 |
}
|
| 122 |
|
| 123 |
-
# If it's not a GitLab URL, proceed with regular HTML parsing
|
| 124 |
response = requests.get(tos_link)
|
| 125 |
response.raise_for_status()
|
| 126 |
|
| 127 |
-
# Parse the HTML content
|
| 128 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 129 |
|
| 130 |
-
# Try to find the content in various ways
|
| 131 |
content = ""
|
| 132 |
|
| 133 |
-
# Check for pre tags which often contain code or raw text displays
|
| 134 |
pre_element = soup.find('pre')
|
| 135 |
if pre_element:
|
| 136 |
content = pre_element.get_text()
|
| 137 |
else:
|
| 138 |
-
# Get the main content if no pre tag
|
| 139 |
main_content = soup.find('main') or soup.find('article') or soup.find('body')
|
| 140 |
if main_content:
|
| 141 |
content = main_content.get_text(strip=True)
|
|
|
|
| 83 |
Dictionary containing company name, link, and extracted content
|
| 84 |
"""
|
| 85 |
|
|
|
|
| 86 |
tos_info = search_terms_of_service(company_name)
|
| 87 |
|
| 88 |
+
# error
|
| 89 |
if "error" in tos_info:
|
| 90 |
return tos_info
|
| 91 |
|
|
|
|
| 95 |
|
| 96 |
print(f"Accessing TOS link: {tos_link}")
|
| 97 |
|
| 98 |
+
# Gitlab repo
|
| 99 |
if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
|
| 100 |
+
# spremeni '/-/blob/' z '/-/raw/'
|
|
|
|
| 101 |
raw_url = tos_link.replace("/-/blob/", "/-/raw/")
|
| 102 |
|
| 103 |
+
|
| 104 |
if "?" in raw_url:
|
| 105 |
raw_url = raw_url.split("?")[0]
|
| 106 |
|
|
|
|
| 118 |
"content_type": "markdown" if raw_url.endswith(".md") else "text"
|
| 119 |
}
|
| 120 |
|
|
|
|
| 121 |
response = requests.get(tos_link)
|
| 122 |
response.raise_for_status()
|
| 123 |
|
|
|
|
| 124 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 125 |
|
|
|
|
| 126 |
content = ""
|
| 127 |
|
|
|
|
| 128 |
pre_element = soup.find('pre')
|
| 129 |
if pre_element:
|
| 130 |
content = pre_element.get_text()
|
| 131 |
else:
|
|
|
|
| 132 |
main_content = soup.find('main') or soup.find('article') or soup.find('body')
|
| 133 |
if main_content:
|
| 134 |
content = main_content.get_text(strip=True)
|