FastAPI-HackathonIn2Mins / src /utils /partner_utils.py
utkarsh134's picture
for pushin
233222c
Raw History Blame Contribute Delete
11 kB
import os
import re
import json
import requests
from datetime import datetime
from typing import Optional, Dict
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import time
from .core import get_llm
def scrape_website(url: str, wait_time: int = 10) -> str:
"""
Scrapes a website using Selenium and returns the HTML content.
"""
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
driver = webdriver.Chrome(
service=Service(ChromeDriverManager().install()),
options=chrome_options
)
try:
driver.get(url)
time.sleep(wait_time)
return driver.page_source
finally:
driver.quit()
def analyze_with_llm(html_content: str, url: str, llm) -> dict:
"""
Process the HTML content with the LLM to extract partner information.
"""
prompt = f"""
Analyze the following HTML content and extract information about the partner or company.
Return ONLY a JSON object of the following format,all the defined fields are mandatory, with no additional text:
{{
"active": true,
"customUrl": "<domain-with-dashes-without-the-.com>",
"partnerDisplayName": "<Readable name of the partner>",
"partnerEstablishmentDate": "<year of establishment of the company or when the company waas founded>",
"partnerLogoPath": "<URL of the partner's logo>",
"partnerName": "<Official Company Name>",
"partnerType": <type of industry inferred from the llm in all caps>,
"partnerDescription": "<A concise and informative description of the partner's business or activities>",
"partnerIndustry": "<Specific industry category the partner operates in>",
"partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}"
// include other social media links like twitter instagram linkedin if present
}}
IMPORTANT: For the "partnerSocialLinks" field, it should be a JSON string (not an object).
For example: "partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}", do the same for other social links if found
If a specific piece of information is not available in the HTML, use an empty string for that field.
Make sure to return valid JSON that can be parsed by Python's json module.
HTML CONTENT:
{html_content}
"""
response = llm.invoke(prompt)
response_text = response.content
try:
if "```json" in response_text and "```" in response_text.split("```json")[1]:
json_content = response_text.split("```json")[1].split("```")[0].strip()
elif "```" in response_text and "```" in response_text.split("```")[1]:
json_content = response_text.split("```")[1].split("```")[0].strip()
else:
json_content = response_text.strip()
return json.loads(json_content)
except Exception as e:
print(f"Error parsing JSON from LLM response: {e}")
domain = url.split('/')[2] if len(url.split('/')) > 2 else url
domain = domain.replace("www.", "")
return {
"active": True,
"customUrl": domain.replace(".", "-"),
"partnerDisplayName": domain.split('.')[0].title(),
"partnerEstablishmentDate": str(datetime.now().year),
"partnerLogoPath": "NA",
"partnerName": domain.split('.')[0].title(),
"partnerType": "INDUSTRY",
"partnerDescription": "Technology company",
"partnerIndustry": "Technology",
"partnerSocialLinks": json.dumps({"websiteurl": url})
}
def extract_partner_info(url: str, llm=None) -> dict:
"""
Extracts partner information from a given URL using Selenium and LLM analysis.
"""
if llm is None:
llm = get_llm()
try:
# Normalize URL
if not url.startswith(('http://', 'https://')):
url = f'https://{url}'
# Scrape website using Selenium
html_content = scrape_website(url)
# Analyze content with LLM
partner_info = analyze_with_llm(html_content, url, llm)
return partner_info
except Exception as e:
print(f"Error in extract_partner_info: {str(e)}")
domain = url.split('/')[2] if len(url.split('/')) > 2 else url
domain = domain.replace("www.", "")
return {
"active": True,
"customUrl": domain.replace(".", "-"),
"partnerDisplayName": domain.split('.')[0].title(),
"partnerEstablishmentDate": str(datetime.now().year),
"partnerLogoPath": "NA",
"partnerName": domain.split('.')[0].title(),
"partnerType": "INDUSTRY",
"partnerDescription": "Technology company",
"partnerIndustry": "Technology",
"partnerSocialLinks": json.dumps({"websiteurl": url})
}
def get_existing_partner(partner_name: str) -> Optional[dict]:
"""Find if partner already exists in the system."""
api_url = os.getenv("PARTNER_URL")
try:
normalized_name = partner_name.lower().strip()
response = requests.get(f"{api_url}?name={partner_name}")
response.raise_for_status()
partners = response.json()
if partners and isinstance(partners, list):
for partner in partners:
if partner.get("partnerName", "").lower().strip() == normalized_name:
return partner
for partner in partners:
if normalized_name in partner.get("partnerName", "").lower() or \
partner.get("partnerName", "").lower() in normalized_name:
return partner
domain_match = re.search(r'(?:https?://)?(?:www\.)?([^/]+)', partner_name)
if domain_match:
domain = domain_match.group(1)
response = requests.get(f"{api_url}")
response.raise_for_status()
all_partners = response.json()
if all_partners and isinstance(all_partners, list):
for partner in all_partners:
partner_links = partner.get("partnerSocialLinks", "{}")
if isinstance(partner_links, str):
try:
links_dict = json.loads(partner_links)
website = links_dict.get("websiteurl", "")
if domain in website:
return partner
except json.JSONDecodeError:
pass
return None
except Exception as e:
print(f"Error checking existing partner: {str(e)}")
return None
def register_partner(partner_info: dict) -> dict:
"""Register a new partner in the system."""
api_url = os.getenv("PARTNER_URL")
headers = {
"Content-Type": "application/json",
"Accept": "application/json"
}
try:
response = requests.post(api_url, json=partner_info, headers=headers)
response.raise_for_status()
return response.json()
except Exception as e:
print(f"\n!!! Error in register_partner: {str(e)}")
raise
def validate_partner_url(url: str) -> bool:
"""
Validates if a given URL is accessible and returns a valid response.
"""
try:
if not url.startswith(('http://', 'https://')):
url = f'https://{url}'
session = requests.Session()
session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
})
response = session.get(
url,
timeout=15,
verify=False,
allow_redirects=True
)
# Check if response is successful and contains HTML content
return (response.status_code == 200 and
'text/html' in response.headers.get('Content-Type', '').lower())
except Exception as e:
print(f"URL validation error: {str(e)}")
return False
def find_partner_by_url(url: str) -> Optional[dict]:
"""
Find a partner by URL in the social links
"""
api_url = os.getenv("PARTNER_URL")
try:
# Normalize the URL for comparison
if not url.startswith(('http://', 'https://')):
url = f'https://{url}'
# Remove trailing slash if present
url = url.rstrip('/')
# Get all partners
response = requests.get(api_url)
response.raise_for_status()
partners = response.json()
if not partners or not isinstance(partners, list):
return None
# First try exact URL match
for partner in partners:
social_links = partner.get("partnerSocialLinks", "{}")
if isinstance(social_links, str):
try:
links_dict = json.loads(social_links)
website = links_dict.get("websiteurl", "").rstrip('/')
# Try different variations of the URL
url_variations = [
url,
url.replace("https://", "http://"),
url.replace("http://", "https://"),
url.replace("www.", ""),
url + "/"
]
if website and any(website == variation for variation in url_variations):
return partner
except json.JSONDecodeError:
pass
# Try domain match if exact match fails
domain = url.split('/')[2] if len(url.split('/')) > 2 else url
domain = domain.replace("www.", "")
for partner in partners:
social_links = partner.get("partnerSocialLinks", "{}")
if isinstance(social_links, str):
try:
links_dict = json.loads(social_links)
website = links_dict.get("websiteurl", "")
if domain in website or domain.split('.')[0] in website:
return partner
except json.JSONDecodeError:
pass
print(f"No partner found for URL: {url}")
return None
except Exception as e:
print(f"Error finding partner by URL: {str(e)}")
return None