Spaces:
Sleeping
Sleeping
Download src/utils/partner_utils.py from utkarsh134/FastAPI-HackathonIn2Mins: direct link, hf CLI and curl.
- Browser
- Download file 11 kB
-
https://huggingface.co/spaces/utkarsh134/FastAPI-HackathonIn2Mins/resolve/main/src/utils/partner_utils.py
- Command line
-
hf download hf://spaces/utkarsh134/FastAPI-HackathonIn2Mins/src/utils/partner_utils.py
-
curl -L -o partner_utils.py https://huggingface.co/spaces/utkarsh134/FastAPI-HackathonIn2Mins/resolve/main/src/utils/partner_utils.py
11 kB
| import os | |
| import re | |
| import json | |
| import requests | |
| from datetime import datetime | |
| from typing import Optional, Dict | |
| from bs4 import BeautifulSoup | |
| from selenium import webdriver | |
| from selenium.webdriver.chrome.service import Service | |
| from selenium.webdriver.chrome.options import Options | |
| from webdriver_manager.chrome import ChromeDriverManager | |
| import time | |
| from .core import get_llm | |
| def scrape_website(url: str, wait_time: int = 10) -> str: | |
| """ | |
| Scrapes a website using Selenium and returns the HTML content. | |
| """ | |
| chrome_options = Options() | |
| chrome_options.add_argument("--headless") | |
| chrome_options.add_argument("--no-sandbox") | |
| chrome_options.add_argument("--disable-dev-shm-usage") | |
| driver = webdriver.Chrome( | |
| service=Service(ChromeDriverManager().install()), | |
| options=chrome_options | |
| ) | |
| try: | |
| driver.get(url) | |
| time.sleep(wait_time) | |
| return driver.page_source | |
| finally: | |
| driver.quit() | |
| def analyze_with_llm(html_content: str, url: str, llm) -> dict: | |
| """ | |
| Process the HTML content with the LLM to extract partner information. | |
| """ | |
| prompt = f""" | |
| Analyze the following HTML content and extract information about the partner or company. | |
| Return ONLY a JSON object of the following format,all the defined fields are mandatory, with no additional text: | |
| {{ | |
| "active": true, | |
| "customUrl": "<domain-with-dashes-without-the-.com>", | |
| "partnerDisplayName": "<Readable name of the partner>", | |
| "partnerEstablishmentDate": "<year of establishment of the company or when the company waas founded>", | |
| "partnerLogoPath": "<URL of the partner's logo>", | |
| "partnerName": "<Official Company Name>", | |
| "partnerType": <type of industry inferred from the llm in all caps>, | |
| "partnerDescription": "<A concise and informative description of the partner's business or activities>", | |
| "partnerIndustry": "<Specific industry category the partner operates in>", | |
| "partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}" | |
| // include other social media links like twitter instagram linkedin if present | |
| }} | |
| IMPORTANT: For the "partnerSocialLinks" field, it should be a JSON string (not an object). | |
| For example: "partnerSocialLinks": "{{\\\"websiteurl\\\": \\\"https://www.example.com/\\\"}}", do the same for other social links if found | |
| If a specific piece of information is not available in the HTML, use an empty string for that field. | |
| Make sure to return valid JSON that can be parsed by Python's json module. | |
| HTML CONTENT: | |
| {html_content} | |
| """ | |
| response = llm.invoke(prompt) | |
| response_text = response.content | |
| try: | |
| if "```json" in response_text and "```" in response_text.split("```json")[1]: | |
| json_content = response_text.split("```json")[1].split("```")[0].strip() | |
| elif "```" in response_text and "```" in response_text.split("```")[1]: | |
| json_content = response_text.split("```")[1].split("```")[0].strip() | |
| else: | |
| json_content = response_text.strip() | |
| return json.loads(json_content) | |
| except Exception as e: | |
| print(f"Error parsing JSON from LLM response: {e}") | |
| domain = url.split('/')[2] if len(url.split('/')) > 2 else url | |
| domain = domain.replace("www.", "") | |
| return { | |
| "active": True, | |
| "customUrl": domain.replace(".", "-"), | |
| "partnerDisplayName": domain.split('.')[0].title(), | |
| "partnerEstablishmentDate": str(datetime.now().year), | |
| "partnerLogoPath": "NA", | |
| "partnerName": domain.split('.')[0].title(), | |
| "partnerType": "INDUSTRY", | |
| "partnerDescription": "Technology company", | |
| "partnerIndustry": "Technology", | |
| "partnerSocialLinks": json.dumps({"websiteurl": url}) | |
| } | |
| def extract_partner_info(url: str, llm=None) -> dict: | |
| """ | |
| Extracts partner information from a given URL using Selenium and LLM analysis. | |
| """ | |
| if llm is None: | |
| llm = get_llm() | |
| try: | |
| # Normalize URL | |
| if not url.startswith(('http://', 'https://')): | |
| url = f'https://{url}' | |
| # Scrape website using Selenium | |
| html_content = scrape_website(url) | |
| # Analyze content with LLM | |
| partner_info = analyze_with_llm(html_content, url, llm) | |
| return partner_info | |
| except Exception as e: | |
| print(f"Error in extract_partner_info: {str(e)}") | |
| domain = url.split('/')[2] if len(url.split('/')) > 2 else url | |
| domain = domain.replace("www.", "") | |
| return { | |
| "active": True, | |
| "customUrl": domain.replace(".", "-"), | |
| "partnerDisplayName": domain.split('.')[0].title(), | |
| "partnerEstablishmentDate": str(datetime.now().year), | |
| "partnerLogoPath": "NA", | |
| "partnerName": domain.split('.')[0].title(), | |
| "partnerType": "INDUSTRY", | |
| "partnerDescription": "Technology company", | |
| "partnerIndustry": "Technology", | |
| "partnerSocialLinks": json.dumps({"websiteurl": url}) | |
| } | |
| def get_existing_partner(partner_name: str) -> Optional[dict]: | |
| """Find if partner already exists in the system.""" | |
| api_url = os.getenv("PARTNER_URL") | |
| try: | |
| normalized_name = partner_name.lower().strip() | |
| response = requests.get(f"{api_url}?name={partner_name}") | |
| response.raise_for_status() | |
| partners = response.json() | |
| if partners and isinstance(partners, list): | |
| for partner in partners: | |
| if partner.get("partnerName", "").lower().strip() == normalized_name: | |
| return partner | |
| for partner in partners: | |
| if normalized_name in partner.get("partnerName", "").lower() or \ | |
| partner.get("partnerName", "").lower() in normalized_name: | |
| return partner | |
| domain_match = re.search(r'(?:https?://)?(?:www\.)?([^/]+)', partner_name) | |
| if domain_match: | |
| domain = domain_match.group(1) | |
| response = requests.get(f"{api_url}") | |
| response.raise_for_status() | |
| all_partners = response.json() | |
| if all_partners and isinstance(all_partners, list): | |
| for partner in all_partners: | |
| partner_links = partner.get("partnerSocialLinks", "{}") | |
| if isinstance(partner_links, str): | |
| try: | |
| links_dict = json.loads(partner_links) | |
| website = links_dict.get("websiteurl", "") | |
| if domain in website: | |
| return partner | |
| except json.JSONDecodeError: | |
| pass | |
| return None | |
| except Exception as e: | |
| print(f"Error checking existing partner: {str(e)}") | |
| return None | |
| def register_partner(partner_info: dict) -> dict: | |
| """Register a new partner in the system.""" | |
| api_url = os.getenv("PARTNER_URL") | |
| headers = { | |
| "Content-Type": "application/json", | |
| "Accept": "application/json" | |
| } | |
| try: | |
| response = requests.post(api_url, json=partner_info, headers=headers) | |
| response.raise_for_status() | |
| return response.json() | |
| except Exception as e: | |
| print(f"\n!!! Error in register_partner: {str(e)}") | |
| raise | |
| def validate_partner_url(url: str) -> bool: | |
| """ | |
| Validates if a given URL is accessible and returns a valid response. | |
| """ | |
| try: | |
| if not url.startswith(('http://', 'https://')): | |
| url = f'https://{url}' | |
| session = requests.Session() | |
| session.headers.update({ | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' | |
| }) | |
| response = session.get( | |
| url, | |
| timeout=15, | |
| verify=False, | |
| allow_redirects=True | |
| ) | |
| # Check if response is successful and contains HTML content | |
| return (response.status_code == 200 and | |
| 'text/html' in response.headers.get('Content-Type', '').lower()) | |
| except Exception as e: | |
| print(f"URL validation error: {str(e)}") | |
| return False | |
| def find_partner_by_url(url: str) -> Optional[dict]: | |
| """ | |
| Find a partner by URL in the social links | |
| """ | |
| api_url = os.getenv("PARTNER_URL") | |
| try: | |
| # Normalize the URL for comparison | |
| if not url.startswith(('http://', 'https://')): | |
| url = f'https://{url}' | |
| # Remove trailing slash if present | |
| url = url.rstrip('/') | |
| # Get all partners | |
| response = requests.get(api_url) | |
| response.raise_for_status() | |
| partners = response.json() | |
| if not partners or not isinstance(partners, list): | |
| return None | |
| # First try exact URL match | |
| for partner in partners: | |
| social_links = partner.get("partnerSocialLinks", "{}") | |
| if isinstance(social_links, str): | |
| try: | |
| links_dict = json.loads(social_links) | |
| website = links_dict.get("websiteurl", "").rstrip('/') | |
| # Try different variations of the URL | |
| url_variations = [ | |
| url, | |
| url.replace("https://", "http://"), | |
| url.replace("http://", "https://"), | |
| url.replace("www.", ""), | |
| url + "/" | |
| ] | |
| if website and any(website == variation for variation in url_variations): | |
| return partner | |
| except json.JSONDecodeError: | |
| pass | |
| # Try domain match if exact match fails | |
| domain = url.split('/')[2] if len(url.split('/')) > 2 else url | |
| domain = domain.replace("www.", "") | |
| for partner in partners: | |
| social_links = partner.get("partnerSocialLinks", "{}") | |
| if isinstance(social_links, str): | |
| try: | |
| links_dict = json.loads(social_links) | |
| website = links_dict.get("websiteurl", "") | |
| if domain in website or domain.split('.')[0] in website: | |
| return partner | |
| except json.JSONDecodeError: | |
| pass | |
| print(f"No partner found for URL: {url}") | |
| return None | |
| except Exception as e: | |
| print(f"Error finding partner by URL: {str(e)}") | |
| return None |