Blame for data_enrichment.py

"""
StrategyStack Multi-API Data Enrichment Module

Integrates multiple RapidAPI sources for comprehensive contact/lead enrichment:
- LinkedIn Data APIs (3 providers)
- Apollo.io
- Social Media (Twitter, Facebook)
- Mediafy API
- Google Search
- LetScrape (Local Business Data + Website Contacts)

Development version with fallback support and multi-provider redundancy.
"""

import os
import requests
import json
import time
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, Optional, List, Any
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


class DataEnrichmentManager:
    """
    Manages data enrichment across multiple API providers.
    Implements fallback logic, caching, and rate limiting.
    """
    
    def __init__(self, username: str):
        self.username = username
        self.cache_dir = Path(f"/data/users/{username}/enrichment_cache")
        self.cache_dir.mkdir(parents=True, exist_ok=True)
        
        # API Keys from environment
        self.api_keys = {
            'linkedin_fresh': os.getenv('RAPIDAPI_KEY_LINKEDIN_FRESH'),
            'linkedin_rockapis': os.getenv('RAPIDAPI_KEY_LINKEDIN_ROCKAPIS'),
            'linkedin_liscraper': os.getenv('RAPIDAPI_KEY_LISCRAPER'),
            'linkedin_scraper': os.getenv('RAPIDAPI_KEY_LINKEDIN_SCRAPER'),
            'apollo': os.getenv('RAPIDAPI_KEY_APOLLO'),
            'mediafy': os.getenv('RAPIDAPI_KEY_MEDIAFY'),
            'twitter': os.getenv('RAPIDAPI_KEY_TWITTER'),
            'facebook': os.getenv('RAPIDAPI_KEY_FACEBOOK'),
            'google': os.getenv('RAPIDAPI_KEY_GOOGLE'),
            'letscrape': os.getenv('RAPIDAPI_KEY_LETSCRAPE')
        }
        
        # RapidAPI host configurations
        self.api_hosts = {
            'linkedin_fresh': 'fresh-linkedin-profile-data.p.rapidapi.com',
            'linkedin_rockapis': 'linkedin-data-api.p.rapidapi.com',
            'linkedin_liscraper': 'li-data-scraper.p.rapidapi.com',
            'linkedin_scraper': 'linkedin-data-scraper.p.rapidapi.com',
            'apollo': 'apollo-io-no-cookies-required.p.rapidapi.com',
            'mediafy': 'mediafy-api.p.rapidapi.com',
            'twitter': 'twitter241.p.rapidapi.com',
            'facebook': 'facebook-scraper3.p.rapidapi.com',
            'google': 'google-search74.p.rapidapi.com',
            'letscrape_business': 'local-business-data.p.rapidapi.com',
            'letscrape_contacts': 'website-contacts-scraper.p.rapidapi.com'
        }
        
        self.cache_ttl = timedelta(days=30)

    
    def _make_request(self, api_name: str, url: str, params: Dict = None, headers: Dict = None) -> Optional[Dict]:
        """Make API request with error handling"""
        try:
            api_key = self.api_keys.get(api_name) or self.api_keys.get('letscrape')
            if not api_key:
                logger.warning(f"API key not found for {api_name}")
                return None
            
            # Determine host
            host = self.api_hosts.get(api_name, '')
            
            request_headers = {
                'x-rapidapi-key': api_key,
                'x-rapidapi-host': host
            }
            if headers:
                request_headers.update(headers)
            
            response = requests.get(url, headers=request_headers, params=params, timeout=30)
            response.raise_for_status()
            return response.json()
        except Exception as e:
            logger.error(f"API request failed for {api_name}: {str(e)}")
            return None
    
    def _get_cache_path(self, identifier: str, data_type: str) -> Path:
        """Get cache file path for identifier"""
        safe_id = identifier.replace('/', '_').replace(':', '_')
        return self.cache_dir / f"{data_type}_{safe_id}.json"
    
    def _load_from_cache(self, identifier: str, data_type: str) -> Optional[Dict]:
        """Load data from cache if valid"""
        cache_path = self._get_cache_path(identifier, data_type)
        if not cache_path.exists():
            return None
        
        try:
            with open(cache_path, 'r') as f:
                cached = json.load(f)
            
            cached_time = datetime.fromisoformat(cached.get('cached_at', '2020-01-01'))
            if datetime.now() - cached_time < self.cache_ttl:
                logger.info(f"Cache hit for {data_type}: {identifier}")
                return cached.get('data')
        except Exception as e:
            logger.error(f"Cache read error: {str(e)}")
        
        return None
    
    def _save_to_cache(self, identifier: str, data_type: str, data: Dict):
        """Save data to cache"""
        cache_path = self._get_cache_path(identifier, data_type)
        try:
            cache_data = {
                'cached_at': datetime.now().isoformat(),
                'identifier': identifier,
                'data': data
            }
            with open(cache_path, 'w') as f:
                json.dump(cache_data, f, indent=2)
            logger.info(f"Cached {data_type}: {identifier}")
        except Exception as e:
            logger.error(f"Cache write error: {str(e)}")

    # LinkedIn Enrichment Methods
    def enrich_linkedin_profile(self, linkedin_url: str) -> Optional[Dict]:
        """Enrich LinkedIn profile with fallback across multiple providers"""
        cached = self._load_from_cache(linkedin_url, 'linkedin')
        if cached:
            return cached
        
        providers = ['linkedin_fresh', 'linkedin_rockapis', 'linkedin_liscraper', 'linkedin_scraper']
        
        for provider in providers:
            try:
                if provider == 'linkedin_fresh':
                    url = 'https://fresh-linkedin-profile-data.p.rapidapi.com/enrich-lead'
                    params = {'linkedin_url': linkedin_url}
                elif provider == 'linkedin_rockapis':
                    url = 'https://linkedin-data-api.p.rapidapi.com/get-profile-data-by-url'
                    params = {'url': linkedin_url}
                elif provider == 'linkedin_liscraper':
                    url = 'https://li-data-scraper.p.rapidapi.com/get-profile-data-by-url'
                    params = {'url': linkedin_url}
                else:
                    url = 'https://linkedin-data-scraper.p.rapidapi.com/person'
                    params = {'url': linkedin_url}
                
                data = self._make_request(provider, url, params)
                if data:
                    self._save_to_cache(linkedin_url, 'linkedin', data)
                    logger.info(f"LinkedIn enrichment successful via {provider}")
                    return data
            except Exception as e:
                logger.warning(f"Provider {provider} failed: {str(e)}")
                continue
        
        return None
    
    def enrich_company(self, domain: str) -> Optional[Dict]:
        """Enrich company data by domain"""
        cached = self._load_from_cache(domain, 'company')
        if cached:
            return cached
        
        url = 'https://linkedin-data-api.p.rapidapi.com/get-company-by-domain'
        params = {'domain': domain}
        data = self._make_request('linkedin_rockapis', url, params)
        
        if data:
            self._save_to_cache(domain, 'company', data)
        return data
    
    # Apollo.io Enrichment
    def enrich_with_apollo(self, email: str = None, domain: str = None) -> Optional[Dict]:
        """Enrich contact using Apollo.io API"""
        cache_key = email or domain
        cached = self._load_from_cache(cache_key, 'apollo')
        if cached:
            return cached
        
        url = 'https://apollo-io-no-cookies-required.p.rapidapi.com/enrich'
        params = {}
        if email:
            params['email'] = email
        if domain:
            params['domain'] = domain
        
        data = self._make_request('apollo', url, params)
        if data:
            self._save_to_cache(cache_key, 'apollo', data)
        return data

    # Social Media Enrichment
    def enrich_twitter_profile(self, username: str) -> Optional[Dict]:
        """Enrich Twitter profile data"""
        cached = self._load_from_cache(username, 'twitter')
        if cached:
            return cached
        
        url = 'https://twitter241.p.rapidapi.com/user'
        params = {'username': username}
        data = self._make_request('twitter', url, params)
        
        if data:
            self._save_to_cache(username, 'twitter', data)
        return data
    
    def enrich_facebook_profile(self, profile_id: str) -> Optional[Dict]:
        """Enrich Facebook profile data"""
        cached = self._load_from_cache(profile_id, 'facebook')
        if cached:
            return cached
        
        url = 'https://facebook-scraper3.p.rapidapi.com/profile'
        params = {'profile_id': profile_id}
        data = self._make_request('facebook', url, params)
        
        if data:
            self._save_to_cache(profile_id, 'facebook', data)
        return data
    
    # Mediafy API
    def enrich_with_mediafy(self, query: str) -> Optional[Dict]:
        """Enrich with Mediafy media/social data"""
        cached = self._load_from_cache(query, 'mediafy')
        if cached:
            return cached
        
        url = 'https://mediafy-api.p.rapidapi.com/search'
        params = {'query': query}
        data = self._make_request('mediafy', url, params)
        
        if data:
            self._save_to_cache(query, 'mediafy', data)
        return data
    
    # Google Search
    def search_google(self, query: str, num_results: int = 10) -> Optional[Dict]:
        """Perform Google search for contact information"""
        cached = self._load_from_cache(query, 'google_search')
        if cached:
            return cached
        
        url = 'https://google-search74.p.rapidapi.com/'
        params = {'query': query, 'limit': str(num_results)}
        data = self._make_request('google', url, params)
        
        if data:
            self._save_to_cache(query, 'google_search', data)
        return data

    # LetScrape APIs - Local Business & Website Contacts
    def scrape_local_business(self, query: str, location: str = None, limit: int = 20) -> Optional[Dict]:
        """
        Scrape local business data from Google Maps.
        Returns business information including address, phone, email, ratings, reviews.
        """
        cache_key = f"{query}_{location or 'any'}"
        cached = self._load_from_cache(cache_key, 'local_business')
        if cached:
            return cached
        
        url = 'https://local-business-data.p.rapidapi.com/search'
        params = {'query': query, 'limit': limit}
        if location:
            params['subtypes'] = location
        
        data = self._make_request('letscrape_business', url, params)
        
        if data:
            self._save_to_cache(cache_key, 'local_business', data)
            logger.info(f"Local business data retrieved for: {query}")
        return data
    
    def scrape_website_contacts(self, domain: str) -> Optional[Dict]:
        """
        Scrape and extract email addresses, phone numbers, and social profile links from a website.
        Supports Facebook, Instagram, TikTok, LinkedIn, Twitter, YouTube, Pinterest, Snapchat, GitHub.
        """
        cached = self._load_from_cache(domain, 'website_contacts')
        if cached:
            return cached
        
        url = 'https://website-contacts-scraper.p.rapidapi.com/scrape-contacts'
        params = {'query': domain}
        
        data = self._make_request('letscrape_contacts', url, params)
        
        if data:
            self._save_to_cache(domain, 'website_contacts', data)
            logger.info(f"Website contacts scraped for: {domain}")
        return data

    # Comprehensive Enrichment
    def enrich_contact_full(self, contact_data: Dict) -> Dict:
        """
        Perform comprehensive enrichment using all available data sources.
        Input contact_data can include: email, linkedin_url, twitter, facebook, domain, name, company
        """
        enriched = {'original': contact_data, 'enrichments': {}, 'sources': []}
        
        # LinkedIn enrichment
        if contact_data.get('linkedin_url'):
            linkedin_data = self.enrich_linkedin_profile(contact_data['linkedin_url'])
            if linkedin_data:
                enriched['enrichments']['linkedin'] = linkedin_data
                enriched['sources'].append('linkedin')
        
        # Apollo enrichment
        if contact_data.get('email') or contact_data.get('domain'):
            apollo_data = self.enrich_with_apollo(
                email=contact_data.get('email'),
                domain=contact_data.get('domain')
            )
            if apollo_data:
                enriched['enrichments']['apollo'] = apollo_data
                enriched['sources'].append('apollo')
        
        # Company enrichment
        if contact_data.get('domain'):
            company_data = self.enrich_company(contact_data['domain'])
            if company_data:
                enriched['enrichments']['company'] = company_data
                enriched['sources'].append('company')
        
        # Website contacts scraping
        if contact_data.get('domain'):
            website_contacts = self.scrape_website_contacts(contact_data['domain'])
            if website_contacts:
                enriched['enrichments']['website_contacts'] = website_contacts
                enriched['sources'].append('website_contacts')
        
        # Local business data (if company name and location provided)
        if contact_data.get('company') and contact_data.get('location'):
            business_data = self.scrape_local_business(
                query=contact_data['company'],
                location=contact_data['location']
            )
            if business_data:
                enriched['enrichments']['local_business'] = business_data
                enriched['sources'].append('local_business')
        
        # Twitter enrichment
        if contact_data.get('twitter_username'):
            twitter_data = self.enrich_twitter_profile(contact_data['twitter_username'])
            if twitter_data:
                enriched['enrichments']['twitter'] = twitter_data
                enriched['sources'].append('twitter')
        
        # Facebook enrichment
        if contact_data.get('facebook_id'):
            facebook_data = self.enrich_facebook_profile(contact_data['facebook_id'])
            if facebook_data:
                enriched['enrichments']['facebook'] = facebook_data
                enriched['sources'].append('facebook')
        
        # Google search for additional info
        if contact_data.get('name'):
            search_query = f"{contact_data['name']}"
            if contact_data.get('company'):
                search_query += f" {contact_data['company']}"
            google_results = self.search_google(search_query, num_results=5)
            if google_results:
                enriched['enrichments']['google_search'] = google_results
                enriched['sources'].append('google')
        
        enriched['enriched_at'] = datetime.now().isoformat()
        enriched['total_sources'] = len(enriched['sources'])
        
        return enriched
    
    def get_enrichment_stats(self) -> Dict:
        """Get cache statistics and API usage"""
        stats = {
            'cache_directory': str(self.cache_dir),
            'cached_profiles': 0,
            'cache_types': {},
            'total_api_calls': 0,
            'cache_hits': 0,
            'cache_misses': 0,
            'total_enrichments': 0,
            'successful_enrichments': 0,
            'failed_enrichments': 0
        }
        
        if self.cache_dir.exists():
            cache_files = list(self.cache_dir.glob('*.json'))
            stats['cached_profiles'] = len(cache_files)
            
            for cache_file in cache_files:
                cache_type = cache_file.stem.split('_')[0]
                stats['cache_types'][cache_type] = stats['cache_types'].get(cache_type, 0) + 1
        
        stats['configured_apis'] = [k for k, v in self.api_keys.items() if v]
        stats['total_apis'] = len(stats['configured_apis'])
        
        return stats


# Convenience functions
def create_enrichment_manager(username: str) -> DataEnrichmentManager:
    """Create a new enrichment manager instance"""
    return DataEnrichmentManager(username)


def enrich_contact(username: str, contact_data: Dict) -> Dict:
    """Quick enrichment for a single contact"""
    manager = DataEnrichmentManager(username)
    return manager.enrich_contact_full(contact_data)