Spaces:
Sleeping
Sleeping
File size: 10,993 Bytes
233222c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 | import os
import re
import json
import requests
from datetime import datetime
from typing import Optional, Dict
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import time
from .core import get_llm
def scrape_website(url: str, wait_time: int = 10) -> str:
"""
Scrapes a website using Selenium and returns the HTML content.
"""
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
driver = webdriver.Chrome(
service=Service(ChromeDriverManager().install()),
options=chrome_options
)
try:
driver.get(url)
time.sleep(wait_time)
return driver.page_source
finally:
driver.quit()
def analyze_with_llm(html_content: str, url: str, llm) -> dict:
"""
Process the HTML content with the LLM to extract partner information.
"""
prompt = f"""
Analyze the following HTML content and extract information about the partner or company.
Return ONLY a JSON object of the following format,all the defined fields are mandatory, with no additional text:
{{
"active": true,
"customUrl": "<domain-with-dashes-without-the-.com>",
"partnerDisplayName": "<Readable name of the partner>",
"partnerEstablishmentDate": "<year of establishment of the company or when the company waas founded>",
"partnerLogoPath": "<URL of the partner's logo>",
"partnerName": "<Official Company Name>",
"partnerType": <type of industry inferred from the llm in all caps>,
"partnerDescription": "<A concise and informative description of the partner's business or activities>",
"partnerIndustry": "<Specific industry category the partner operates in>",
"partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}"
// include other social media links like twitter instagram linkedin if present
}}
IMPORTANT: For the "partnerSocialLinks" field, it should be a JSON string (not an object).
For example: "partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}", do the same for other social links if found
If a specific piece of information is not available in the HTML, use an empty string for that field.
Make sure to return valid JSON that can be parsed by Python's json module.
HTML CONTENT:
{html_content}
"""
response = llm.invoke(prompt)
response_text = response.content
try:
if "```json" in response_text and "```" in response_text.split("```json")[1]:
json_content = response_text.split("```json")[1].split("```")[0].strip()
elif "```" in response_text and "```" in response_text.split("```")[1]:
json_content = response_text.split("```")[1].split("```")[0].strip()
else:
json_content = response_text.strip()
return json.loads(json_content)
except Exception as e:
print(f"Error parsing JSON from LLM response: {e}")
domain = url.split('/')[2] if len(url.split('/')) > 2 else url
domain = domain.replace("www.", "")
return {
"active": True,
"customUrl": domain.replace(".", "-"),
"partnerDisplayName": domain.split('.')[0].title(),
"partnerEstablishmentDate": str(datetime.now().year),
"partnerLogoPath": "NA",
"partnerName": domain.split('.')[0].title(),
"partnerType": "INDUSTRY",
"partnerDescription": "Technology company",
"partnerIndustry": "Technology",
"partnerSocialLinks": json.dumps({"websiteurl": url})
}
def extract_partner_info(url: str, llm=None) -> dict:
"""
Extracts partner information from a given URL using Selenium and LLM analysis.
"""
if llm is None:
llm = get_llm()
try:
# Normalize URL
if not url.startswith(('http://', 'https://')):
url = f'https://{url}'
# Scrape website using Selenium
html_content = scrape_website(url)
# Analyze content with LLM
partner_info = analyze_with_llm(html_content, url, llm)
return partner_info
except Exception as e:
print(f"Error in extract_partner_info: {str(e)}")
domain = url.split('/')[2] if len(url.split('/')) > 2 else url
domain = domain.replace("www.", "")
return {
"active": True,
"customUrl": domain.replace(".", "-"),
"partnerDisplayName": domain.split('.')[0].title(),
"partnerEstablishmentDate": str(datetime.now().year),
"partnerLogoPath": "NA",
"partnerName": domain.split('.')[0].title(),
"partnerType": "INDUSTRY",
"partnerDescription": "Technology company",
"partnerIndustry": "Technology",
"partnerSocialLinks": json.dumps({"websiteurl": url})
}
def get_existing_partner(partner_name: str) -> Optional[dict]:
"""Find if partner already exists in the system."""
api_url = os.getenv("PARTNER_URL")
try:
normalized_name = partner_name.lower().strip()
response = requests.get(f"{api_url}?name={partner_name}")
response.raise_for_status()
partners = response.json()
if partners and isinstance(partners, list):
for partner in partners:
if partner.get("partnerName", "").lower().strip() == normalized_name:
return partner
for partner in partners:
if normalized_name in partner.get("partnerName", "").lower() or \
partner.get("partnerName", "").lower() in normalized_name:
return partner
domain_match = re.search(r'(?:https?://)?(?:www\.)?([^/]+)', partner_name)
if domain_match:
domain = domain_match.group(1)
response = requests.get(f"{api_url}")
response.raise_for_status()
all_partners = response.json()
if all_partners and isinstance(all_partners, list):
for partner in all_partners:
partner_links = partner.get("partnerSocialLinks", "{}")
if isinstance(partner_links, str):
try:
links_dict = json.loads(partner_links)
website = links_dict.get("websiteurl", "")
if domain in website:
return partner
except json.JSONDecodeError:
pass
return None
except Exception as e:
print(f"Error checking existing partner: {str(e)}")
return None
def register_partner(partner_info: dict) -> dict:
"""Register a new partner in the system."""
api_url = os.getenv("PARTNER_URL")
headers = {
"Content-Type": "application/json",
"Accept": "application/json"
}
try:
response = requests.post(api_url, json=partner_info, headers=headers)
response.raise_for_status()
return response.json()
except Exception as e:
print(f"\n!!! Error in register_partner: {str(e)}")
raise
def validate_partner_url(url: str) -> bool:
"""
Validates if a given URL is accessible and returns a valid response.
"""
try:
if not url.startswith(('http://', 'https://')):
url = f'https://{url}'
session = requests.Session()
session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
})
response = session.get(
url,
timeout=15,
verify=False,
allow_redirects=True
)
# Check if response is successful and contains HTML content
return (response.status_code == 200 and
'text/html' in response.headers.get('Content-Type', '').lower())
except Exception as e:
print(f"URL validation error: {str(e)}")
return False
def find_partner_by_url(url: str) -> Optional[dict]:
"""
Find a partner by URL in the social links
"""
api_url = os.getenv("PARTNER_URL")
try:
# Normalize the URL for comparison
if not url.startswith(('http://', 'https://')):
url = f'https://{url}'
# Remove trailing slash if present
url = url.rstrip('/')
# Get all partners
response = requests.get(api_url)
response.raise_for_status()
partners = response.json()
if not partners or not isinstance(partners, list):
return None
# First try exact URL match
for partner in partners:
social_links = partner.get("partnerSocialLinks", "{}")
if isinstance(social_links, str):
try:
links_dict = json.loads(social_links)
website = links_dict.get("websiteurl", "").rstrip('/')
# Try different variations of the URL
url_variations = [
url,
url.replace("https://", "http://"),
url.replace("http://", "https://"),
url.replace("www.", ""),
url + "/"
]
if website and any(website == variation for variation in url_variations):
return partner
except json.JSONDecodeError:
pass
# Try domain match if exact match fails
domain = url.split('/')[2] if len(url.split('/')) > 2 else url
domain = domain.replace("www.", "")
for partner in partners:
social_links = partner.get("partnerSocialLinks", "{}")
if isinstance(social_links, str):
try:
links_dict = json.loads(social_links)
website = links_dict.get("websiteurl", "")
if domain in website or domain.split('.')[0] in website:
return partner
except json.JSONDecodeError:
pass
print(f"No partner found for URL: {url}")
return None
except Exception as e:
print(f"Error finding partner by URL: {str(e)}")
return None |