Add multi-API data enrichment module (dev)

data_enrichment.py

@@ -0,0 +1 @@
+"""\nStrategyStack Multi-API Data Enrichment Module\n\nIntegrates multiple RapidAPI sources for comprehensive contact/lead enrichment:\n- LinkedIn Data APIs (3 providers)\n- Apollo.io\n- Social Media (Twitter, Facebook)\n- Mediafy API\n- Google Search\n\nDevelopment version with fallback support and multi-provider redundancy.\n"""\n\nimport os\nimport requests\nimport json\nimport time\nfrom datetime import datetime, timedelta\nfrom pathlib import Path\nfrom typing import Dict, Optional, List, Any\nimport logging\n\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n\nclass DataEnrichmentManager:\n    """\n    Manages data enrichment across multiple API providers.\n    Implements fallback logic, caching, and rate limiting.\n    """\n    \n    def __init__(self, username: str):\n        self.username = username\n        self.cache_dir = Path(f"/data/users/{username}/enrichment_cache")\n        self.cache_dir.mkdir(parents=True, exist_ok=True)\n        \n        # API Keys from environment\n        self.api_keys = {\n            'linkedin_fresh': os.getenv('RAPIDAPI_KEY_LINKEDIN_FRESH'),\n            'linkedin_rockapis': os.getenv('RAPIDAPI_KEY_LINKEDIN_ROCKAPIS'),\n            'linkedin_liscraper': os.getenv('RAPIDAPI_KEY_LISCRAPER'),\n            'linkedin_scraper': os.getenv('RAPIDAPI_KEY_LINKEDIN_SCRAPER'),\n            'apollo': os.getenv('RAPIDAPI_KEY_APOLLO'),\n            'mediafy': os.getenv('RAPIDAPI_KEY_MEDIAFY'),\n            'twitter': os.getenv('RAPIDAPI_KEY_TWITTER'),\n            'facebook': os.getenv('RAPIDAPI_KEY_FACEBOOK'),\n            'google': os.getenv('RAPIDAPI_KEY_GOOGLE')\n        }\n        \n        # RapidAPI host configurations\n        self.api_hosts = {\n            'linkedin_fresh': 'fresh-linkedin-profile-data.p.rapidapi.com',\n            'linkedin_rockapis': 'linkedin-data-api.p.rapidapi.com',\n            'linkedin_liscraper': 'li-data-scraper.p.rapidapi.com',\n            'linkedin_scraper': 'linkedin-data-scraper.p.rapidapi.com',\n            'apollo': 'apollo-io-no-cookies-required.p.rapidapi.com',\n            'mediafy': 'mediafy-api.p.rapidapi.com',\n            'twitter': 'twitter241.p.rapidapi.com',\n            'facebook': 'facebook-scraper3.p.rapidapi.com',\n            'google': 'google-search74.p.rapidapi.com'\n        }\n        \n        self.cache_ttl = timedelta(days=30)\n\n    \n    def _make_request(self, api_name: str, url: str, params: Dict = None, headers: Dict = None) -> Optional[Dict]:\n        """Make API request with error handling"""\n        try:\n            api_key = self.api_keys.get(api_name)\n            if not api_key:\n                logger.warning(f"API key not found for {api_name}")\n                return None\n            \n            request_headers = {\n                'x-rapidapi-key': api_key,\n                'x-rapidapi-host': self.api_hosts.get(api_name, '')\n            }\n            if headers:\n                request_headers.update(headers)\n            \n            response = requests.get(url, headers=request_headers, params=params, timeout=30)\n            response.raise_for_status()\n            return response.json()\n        except Exception as e:\n            logger.error(f"API request failed for {api_name}: {str(e)}")\n            return None\n    \n    def _get_cache_path(self, identifier: str, data_type: str) -> Path:\n        """Get cache file path for identifier"""\n        safe_id = identifier.replace('/', '_').replace(':', '_')\n        return self.cache_dir / f"{data_type}_{safe_id}.json"\n    \n    def _load_from_cache(self, identifier: str, data_type: str) -> Optional[Dict]:\n        """Load data from cache if valid"""\n        cache_path = self._get_cache_path(identifier, data_type)\n        if not cache_path.exists():\n            return None\n        \n        try:\n            with open(cache_path, 'r') as f:\n                cached = json.load(f)\n            \n            cached_time = datetime.fromisoformat(cached.get('cached_at', '2020-01-01'))\n            if datetime.now() - cached_time < self.cache_ttl:\n                logger.info(f"Cache hit for {data_type}: {identifier}")\n                return cached.get('data')\n        except Exception as e:\n            logger.error(f"Cache read error: {str(e)}")\n        \n        return None\n    \n    def _save_to_cache(self, identifier: str, data_type: str, data: Dict):\n        """Save data to cache"""\n        cache_path = self._get_cache_path(identifier, data_type)\n        try:\n            cache_data = {\n                'cached_at': datetime.now().isoformat(),\n                'identifier': identifier,\n                'data': data\n            }\n            with open(cache_path, 'w') as f:\n                json.dump(cache_data, f, indent=2)\n            logger.info(f"Cached {data_type}: {identifier}")\n        except Exception as e:\n            logger.error(f"Cache write error: {str(e)}")\n\n    # LinkedIn Enrichment Methods\n    def enrich_linkedin_profile(self, linkedin_url: str) -> Optional[Dict]:\n        """Enrich LinkedIn profile with fallback across multiple providers"""\n        # Check cache first\n        cached = self._load_from_cache(linkedin_url, 'linkedin')\n        if cached:\n            return cached\n        \n        providers = ['linkedin_fresh', 'linkedin_rockapis', 'linkedin_liscraper', 'linkedin_scraper']\n        \n        for provider in providers:\n            try:\n                if provider == 'linkedin_fresh':\n                    url = 'https://fresh-linkedin-profile-data.p.rapidapi.com/enrich-lead'\n                    params = {'linkedin_url': linkedin_url}\n                elif provider == 'linkedin_rockapis':\n                    url = 'https://linkedin-data-api.p.rapidapi.com/get-profile-data-by-url'\n                    params = {'url': linkedin_url}\n                elif provider == 'linkedin_liscraper':\n                    url = 'https://li-data-scraper.p.rapidapi.com/get-profile-data-by-url'\n                    params = {'url': linkedin_url}\n                else:  # linkedin_scraper\n                    url = 'https://linkedin-data-scraper.p.rapidapi.com/person'\n                    params = {'url': linkedin_url}\n                \n                data = self._make_request(provider, url, params)\n                if data:\n                    self._save_to_cache(linkedin_url, 'linkedin', data)\n                    logger.info(f"LinkedIn enrichment successful via {provider}")\n                    return data\n            except Exception as e:\n                logger.warning(f"Provider {provider} failed: {str(e)}")\n                continue\n        \n        return None\n    \n    def enrich_company(self, domain: str) -> Optional[Dict]:\n        """Enrich company data by domain"""\n        cached = self._load_from_cache(domain, 'company')\n        if cached:\n            return cached\n        \n        url = 'https://linkedin-data-api.p.rapidapi.com/get-company-by-domain'\n        params = {'domain': domain}\n        data = self._make_request('linkedin_rockapis', url, params)\n        \n        if data:\n            self._save_to_cache(domain, 'company', data)\n        return data\n\n    # Apollo.io Enrichment\n    def enrich_with_apollo(self, email: str = None, domain: str = None) -> Optional[Dict]:\n        """Enrich contact using Apollo.io API"""\n        cache_key = email or domain\n        cached = self._load_from_cache(cache_key, 'apollo')\n        if cached:\n            return cached\n        \n        url = 'https://apollo-io-no-cookies-required.p.rapidapi.com/enrich'\n        params = {}\n        if email:\n            params['email'] = email\n        if domain:\n            params['domain'] = domain\n        \n        data = self._make_request('apollo', url, params)\n        if data:\n            self._save_to_cache(cache_key, 'apollo', data)\n        return data\n    \n    # Social Media Enrichment\n    def enrich_twitter_profile(self, username: str) -> Optional[Dict]:\n        """Enrich Twitter profile data"""\n        cached = self._load_from_cache(username, 'twitter')\n        if cached:\n            return cached\n        \n        url = f'https://twitter241.p.rapidapi.com/user'\n        params = {'username': username}\n        data = self._make_request('twitter', url, params)\n        \n        if data:\n            self._save_to_cache(username, 'twitter', data)\n        return data\n    \n    def enrich_facebook_profile(self, profile_id: str) -> Optional[Dict]:\n        """Enrich Facebook profile data"""\n        cached = self._load_from_cache(profile_id, 'facebook')\n        if cached:\n            return cached\n        \n        url = 'https://facebook-scraper3.p.rapidapi.com/profile'\n        params = {'profile_id': profile_id}\n        data = self._make_request('facebook', url, params)\n        \n        if data:\n            self._save_to_cache(profile_id, 'facebook', data)\n        return data\n    \n    # Mediafy API\n    def enrich_with_mediafy(self, query: str) -> Optional[Dict]:\n        """Enrich with Mediafy media/social data"""\n        cached = self._load_from_cache(query, 'mediafy')\n        if cached:\n            return cached\n        \n        url = 'https://mediafy-api.p.rapidapi.com/search'\n        params = {'query': query}\n        data = self._make_request('mediafy', url, params)\n        \n        if data:\n            self._save_to_cache(query, 'mediafy', data)\n        return data\n    \n    # Google Search\n    def search_google(self, query: str, num_results: int = 10) -> Optional[Dict]:\n        """Perform Google search for contact information"""\n        cached = self._load_from_cache(query, 'google_search')\n        if cached:\n            return cached\n        \n        url = 'https://google-search74.p.rapidapi.com/'\n        params = {'query': query, 'limit': str(num_results)}\n        data = self._make_request('google', url, params)\n        \n        if data:\n            self._save_to_cache(query, 'google_search', data)\n        return data\n\n    # Comprehensive Enrichment\n    def enrich_contact_full(self, contact_data: Dict) -> Dict:\n        """\n        Perform comprehensive enrichment using all available data sources.\n        Input contact_data can include: email, linkedin_url, twitter, facebook, domain, name\n        """\n        enriched = {'original': contact_data, 'enrichments': {}, 'sources': []}\n        \n        # LinkedIn enrichment\n        if contact_data.get('linkedin_url'):\n            linkedin_data = self.enrich_linkedin_profile(contact_data['linkedin_url'])\n            if linkedin_data:\n                enriched['enrichments']['linkedin'] = linkedin_data\n                enriched['sources'].append('linkedin')\n        \n        # Apollo enrichment\n        if contact_data.get('email') or contact_data.get('domain'):\n            apollo_data = self.enrich_with_apollo(\n                email=contact_data.get('email'),\n                domain=contact_data.get('domain')\n            )\n            if apollo_data:\n                enriched['enrichments']['apollo'] = apollo_data\n                enriched['sources'].append('apollo')\n        \n        # Company enrichment\n        if contact_data.get('domain'):\n            company_data = self.enrich_company(contact_data['domain'])\n            if company_data:\n                enriched['enrichments']['company'] = company_data\n                enriched['sources'].append('company')\n        \n        # Twitter enrichment\n        if contact_data.get('twitter_username'):\n            twitter_data = self.enrich_twitter_profile(contact_data['twitter_username'])\n            if twitter_data:\n                enriched['enrichments']['twitter'] = twitter_data\n                enriched['sources'].append('twitter')\n        \n        # Facebook enrichment\n        if contact_data.get('facebook_id'):\n            facebook_data = self.enrich_facebook_profile(contact_data['facebook_id'])\n            if facebook_data:\n                enriched['enrichments']['facebook'] = facebook_data\n                enriched['sources'].append('facebook')\n        \n        # Google search for additional info\n        if contact_data.get('name'):\n            search_query = f\"{contact_data['name']}\"\n            if contact_data.get('company'):\n                search_query += f\" {contact_data['company']}\"\n            google_results = self.search_google(search_query, num_results=5)\n            if google_results:\n                enriched['enrichments']['google_search'] = google_results\n                enriched['sources'].append('google')\n        \n        enriched['enriched_at'] = datetime.now().isoformat()\n        enriched['total_sources'] = len(enriched['sources'])\n        \n        return enriched\n    \n    def get_enrichment_stats(self) -> Dict:\n        """Get cache statistics and API usage"""\n        stats = {\n            'cache_directory': str(self.cache_dir),\n            'cached_profiles': 0,\n            'cache_types': {}\n        }\n        \n        if self.cache_dir.exists():\n            cache_files = list(self.cache_dir.glob('*.json'))\n            stats['cached_profiles'] = len(cache_files)\n            \n            for cache_file in cache_files:\n                cache_type = cache_file.stem.split('_')[0]\n                stats['cache_types'][cache_type] = stats['cache_types'].get(cache_type, 0) + 1\n        \n        stats['configured_apis'] = [k for k, v in self.api_keys.items() if v]\n        stats['total_apis'] = len(stats['configured_apis'])\n        \n        return stats\n\n\n# Convenience functions\ndef create_enrichment_manager(username: str) -> DataEnrichmentManager:\n    """Create a new enrichment manager instance"""\n    return DataEnrichmentManager(username)\n\n\ndef enrich_contact(username: str, contact_data: Dict) -> Dict:\n    """Quick enrichment for a single contact"""\n    manager = DataEnrichmentManager(username)\n    return manager.enrich_contact_full(contact_data)\n