import os import re import json import requests from datetime import datetime from typing import Optional, Dict from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.options import Options from webdriver_manager.chrome import ChromeDriverManager import time from .core import get_llm def scrape_website(url: str, wait_time: int = 10) -> str: """ Scrapes a website using Selenium and returns the HTML content. """ chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome( service=Service(ChromeDriverManager().install()), options=chrome_options ) try: driver.get(url) time.sleep(wait_time) return driver.page_source finally: driver.quit() def analyze_with_llm(html_content: str, url: str, llm) -> dict: """ Process the HTML content with the LLM to extract partner information. """ prompt = f""" Analyze the following HTML content and extract information about the partner or company. Return ONLY a JSON object of the following format,all the defined fields are mandatory, with no additional text: {{ "active": true, "customUrl": "", "partnerDisplayName": "", "partnerEstablishmentDate": "", "partnerLogoPath": "", "partnerName": "", "partnerType": , "partnerDescription": "", "partnerIndustry": "", "partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}" // include other social media links like twitter instagram linkedin if present }} IMPORTANT: For the "partnerSocialLinks" field, it should be a JSON string (not an object). For example: "partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}", do the same for other social links if found If a specific piece of information is not available in the HTML, use an empty string for that field. Make sure to return valid JSON that can be parsed by Python's json module. HTML CONTENT: {html_content} """ response = llm.invoke(prompt) response_text = response.content try: if "```json" in response_text and "```" in response_text.split("```json")[1]: json_content = response_text.split("```json")[1].split("```")[0].strip() elif "```" in response_text and "```" in response_text.split("```")[1]: json_content = response_text.split("```")[1].split("```")[0].strip() else: json_content = response_text.strip() return json.loads(json_content) except Exception as e: print(f"Error parsing JSON from LLM response: {e}") domain = url.split('/')[2] if len(url.split('/')) > 2 else url domain = domain.replace("www.", "") return { "active": True, "customUrl": domain.replace(".", "-"), "partnerDisplayName": domain.split('.')[0].title(), "partnerEstablishmentDate": str(datetime.now().year), "partnerLogoPath": "NA", "partnerName": domain.split('.')[0].title(), "partnerType": "INDUSTRY", "partnerDescription": "Technology company", "partnerIndustry": "Technology", "partnerSocialLinks": json.dumps({"websiteurl": url}) } def extract_partner_info(url: str, llm=None) -> dict: """ Extracts partner information from a given URL using Selenium and LLM analysis. """ if llm is None: llm = get_llm() try: # Normalize URL if not url.startswith(('http://', 'https://')): url = f'https://{url}' # Scrape website using Selenium html_content = scrape_website(url) # Analyze content with LLM partner_info = analyze_with_llm(html_content, url, llm) return partner_info except Exception as e: print(f"Error in extract_partner_info: {str(e)}") domain = url.split('/')[2] if len(url.split('/')) > 2 else url domain = domain.replace("www.", "") return { "active": True, "customUrl": domain.replace(".", "-"), "partnerDisplayName": domain.split('.')[0].title(), "partnerEstablishmentDate": str(datetime.now().year), "partnerLogoPath": "NA", "partnerName": domain.split('.')[0].title(), "partnerType": "INDUSTRY", "partnerDescription": "Technology company", "partnerIndustry": "Technology", "partnerSocialLinks": json.dumps({"websiteurl": url}) } def get_existing_partner(partner_name: str) -> Optional[dict]: """Find if partner already exists in the system.""" api_url = os.getenv("PARTNER_URL") try: normalized_name = partner_name.lower().strip() response = requests.get(f"{api_url}?name={partner_name}") response.raise_for_status() partners = response.json() if partners and isinstance(partners, list): for partner in partners: if partner.get("partnerName", "").lower().strip() == normalized_name: return partner for partner in partners: if normalized_name in partner.get("partnerName", "").lower() or \ partner.get("partnerName", "").lower() in normalized_name: return partner domain_match = re.search(r'(?:https?://)?(?:www\.)?([^/]+)', partner_name) if domain_match: domain = domain_match.group(1) response = requests.get(f"{api_url}") response.raise_for_status() all_partners = response.json() if all_partners and isinstance(all_partners, list): for partner in all_partners: partner_links = partner.get("partnerSocialLinks", "{}") if isinstance(partner_links, str): try: links_dict = json.loads(partner_links) website = links_dict.get("websiteurl", "") if domain in website: return partner except json.JSONDecodeError: pass return None except Exception as e: print(f"Error checking existing partner: {str(e)}") return None def register_partner(partner_info: dict) -> dict: """Register a new partner in the system.""" api_url = os.getenv("PARTNER_URL") headers = { "Content-Type": "application/json", "Accept": "application/json" } try: response = requests.post(api_url, json=partner_info, headers=headers) response.raise_for_status() return response.json() except Exception as e: print(f"\n!!! Error in register_partner: {str(e)}") raise def validate_partner_url(url: str) -> bool: """ Validates if a given URL is accessible and returns a valid response. """ try: if not url.startswith(('http://', 'https://')): url = f'https://{url}' session = requests.Session() session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' }) response = session.get( url, timeout=15, verify=False, allow_redirects=True ) # Check if response is successful and contains HTML content return (response.status_code == 200 and 'text/html' in response.headers.get('Content-Type', '').lower()) except Exception as e: print(f"URL validation error: {str(e)}") return False def find_partner_by_url(url: str) -> Optional[dict]: """ Find a partner by URL in the social links """ api_url = os.getenv("PARTNER_URL") try: # Normalize the URL for comparison if not url.startswith(('http://', 'https://')): url = f'https://{url}' # Remove trailing slash if present url = url.rstrip('/') # Get all partners response = requests.get(api_url) response.raise_for_status() partners = response.json() if not partners or not isinstance(partners, list): return None # First try exact URL match for partner in partners: social_links = partner.get("partnerSocialLinks", "{}") if isinstance(social_links, str): try: links_dict = json.loads(social_links) website = links_dict.get("websiteurl", "").rstrip('/') # Try different variations of the URL url_variations = [ url, url.replace("https://", "http://"), url.replace("http://", "https://"), url.replace("www.", ""), url + "/" ] if website and any(website == variation for variation in url_variations): return partner except json.JSONDecodeError: pass # Try domain match if exact match fails domain = url.split('/')[2] if len(url.split('/')) > 2 else url domain = domain.replace("www.", "") for partner in partners: social_links = partner.get("partnerSocialLinks", "{}") if isinstance(social_links, str): try: links_dict = json.loads(social_links) website = links_dict.get("websiteurl", "") if domain in website or domain.split('.')[0] in website: return partner except json.JSONDecodeError: pass print(f"No partner found for URL: {url}") return None except Exception as e: print(f"Error finding partner by URL: {str(e)}") return None