Frog215 commited on
Commit
47c1544
·
verified ·
1 Parent(s): bd73fbf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +39 -28
app.py CHANGED
@@ -74,7 +74,7 @@ def search_terms_of_service(company_name: str) -> Dict:
74
  @tool
75
  def extract_tos_content(company_name: str) -> Dict:
76
  """
77
- Search for a company's Terms of Service, access the link, and extract content.
78
 
79
  Args:
80
  company_name: Name of the organization/company to search for
@@ -96,44 +96,55 @@ def extract_tos_content(company_name: str) -> Dict:
96
 
97
  print(f"Accessing TOS link: {tos_link}")
98
 
99
- # Request the content from the TOS link
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  response = requests.get(tos_link)
101
  response.raise_for_status()
102
 
103
  # Parse the HTML content
104
  soup = BeautifulSoup(response.text, 'html.parser')
105
 
106
- # Extract repository content - assuming it's in a specific element
107
- # You may need to adjust these selectors based on the actual page structure
108
- repo_content = ""
109
- repo_element = soup.find('div', class_='repository-content')
110
- if repo_element:
111
- repo_content = repo_element.get_text(strip=True)
112
-
113
- # Extract TOS content - assuming it's in a specific element
114
- # You may need to adjust these selectors based on the actual page structure
115
- tos_content = ""
116
- tos_element = soup.find('div', id='terms-of-service') or soup.find('div', class_='terms-of-service')
117
- if tos_element:
118
- tos_content = tos_element.get_text(strip=True)
119
-
120
- # If we couldn't find specific elements, try to get the main content
121
- if not repo_content and not tos_content:
122
- main_content = soup.find('main') or soup.find('body')
123
  if main_content:
124
- main_content_text = main_content.get_text(strip=True)
125
- return {
126
- "company_name": tos_info["company_name"],
127
- "tos_link": tos_link,
128
- "content": main_content_text[:5000], # Limit content length
129
- "note": "Could not identify specific repository or TOS sections"
130
- }
131
 
132
  return {
133
  "company_name": tos_info["company_name"],
134
  "tos_link": tos_link,
135
- "repository_content": repo_content[:2500] if repo_content else "No repository content found",
136
- "tos_content": tos_content[:2500] if tos_content else "No Terms of Service content found"
137
  }
138
 
139
  except requests.exceptions.RequestException as e:
 
74
  @tool
75
  def extract_tos_content(company_name: str) -> Dict:
76
  """
77
+ Search for a company's Terms of Service, access the link, and extract raw content from GitLab repository.
78
 
79
  Args:
80
  company_name: Name of the organization/company to search for
 
96
 
97
  print(f"Accessing TOS link: {tos_link}")
98
 
99
+ # Check if it's a GitLab repository URL
100
+ if "code.europa.eu" in tos_link and "/-/blob/" in tos_link:
101
+ # Convert the URL to get the raw content
102
+ # For GitLab, we need to replace '/-/blob/' with '/-/raw/'
103
+ raw_url = tos_link.replace("/-/blob/", "/-/raw/")
104
+
105
+ # Remove any query parameters
106
+ if "?" in raw_url:
107
+ raw_url = raw_url.split("?")[0]
108
+
109
+ print(f"Accessing raw content URL: {raw_url}")
110
+
111
+ # Request the raw content
112
+ response = requests.get(raw_url)
113
+ response.raise_for_status()
114
+
115
+ return {
116
+ "company_name": tos_info["company_name"],
117
+ "tos_link": tos_link,
118
+ "raw_url": raw_url,
119
+ "content": response.text, # Full raw content
120
+ "content_type": "markdown" if raw_url.endswith(".md") else "text"
121
+ }
122
+
123
+ # If it's not a GitLab URL, proceed with regular HTML parsing
124
  response = requests.get(tos_link)
125
  response.raise_for_status()
126
 
127
  # Parse the HTML content
128
  soup = BeautifulSoup(response.text, 'html.parser')
129
 
130
+ # Try to find the content in various ways
131
+ content = ""
132
+
133
+ # Check for pre tags which often contain code or raw text displays
134
+ pre_element = soup.find('pre')
135
+ if pre_element:
136
+ content = pre_element.get_text()
137
+ else:
138
+ # Get the main content if no pre tag
139
+ main_content = soup.find('main') or soup.find('article') or soup.find('body')
 
 
 
 
 
 
 
140
  if main_content:
141
+ content = main_content.get_text(strip=True)
 
 
 
 
 
 
142
 
143
  return {
144
  "company_name": tos_info["company_name"],
145
  "tos_link": tos_link,
146
+ "content": content,
147
+ "note": "Content extracted from HTML page"
148
  }
149
 
150
  except requests.exceptions.RequestException as e: