Frog215 commited on
Commit
a6e113d
·
verified ·
1 Parent(s): 47c1544

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -11
app.py CHANGED
@@ -83,10 +83,9 @@ def extract_tos_content(company_name: str) -> Dict:
83
  Dictionary containing company name, link, and extracted content
84
  """
85
 
86
- # First use the existing function to get the TOS link
87
  tos_info = search_terms_of_service(company_name)
88
 
89
- # If there was an error in the search function, return that error
90
  if "error" in tos_info:
91
  return tos_info
92
 
@@ -96,13 +95,12 @@ def extract_tos_content(company_name: str) -> Dict:
96
 
97
  print(f"Accessing TOS link: {tos_link}")
98
 
99
- # Check if it's a GitLab repository URL
100
  if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
101
- # Convert the URL to get the raw content
102
- # For GitLab, we need to replace '/-/blob/' with '/-/raw/'
103
  raw_url = tos_link.replace("/-/blob/", "/-/raw/")
104
 
105
- # Remove any query parameters
106
  if "?" in raw_url:
107
  raw_url = raw_url.split("?")[0]
108
 
@@ -120,22 +118,17 @@ def extract_tos_content(company_name: str) -> Dict:
120
  "content_type": "markdown" if raw_url.endswith(".md") else "text"
121
  }
122
 
123
- # If it's not a GitLab URL, proceed with regular HTML parsing
124
  response = requests.get(tos_link)
125
  response.raise_for_status()
126
 
127
- # Parse the HTML content
128
  soup = BeautifulSoup(response.text, 'html.parser')
129
 
130
- # Try to find the content in various ways
131
  content = ""
132
 
133
- # Check for pre tags which often contain code or raw text displays
134
  pre_element = soup.find('pre')
135
  if pre_element:
136
  content = pre_element.get_text()
137
  else:
138
- # Get the main content if no pre tag
139
  main_content = soup.find('main') or soup.find('article') or soup.find('body')
140
  if main_content:
141
  content = main_content.get_text(strip=True)
 
83
  Dictionary containing company name, link, and extracted content
84
  """
85
 
 
86
  tos_info = search_terms_of_service(company_name)
87
 
88
+ # error
89
  if "error" in tos_info:
90
  return tos_info
91
 
 
95
 
96
  print(f"Accessing TOS link: {tos_link}")
97
 
98
+ # Gitlab repo
99
  if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
100
+ # spremeni '/-/blob/' z '/-/raw/'
 
101
  raw_url = tos_link.replace("/-/blob/", "/-/raw/")
102
 
103
+
104
  if "?" in raw_url:
105
  raw_url = raw_url.split("?")[0]
106
 
 
118
  "content_type": "markdown" if raw_url.endswith(".md") else "text"
119
  }
120
 
 
121
  response = requests.get(tos_link)
122
  response.raise_for_status()
123
 
 
124
  soup = BeautifulSoup(response.text, 'html.parser')
125
 
 
126
  content = ""
127
 
 
128
  pre_element = soup.find('pre')
129
  if pre_element:
130
  content = pre_element.get_text()
131
  else:
 
132
  main_content = soup.find('main') or soup.find('article') or soup.find('body')
133
  if main_content:
134
  content = main_content.get_text(strip=True)