Delete server/internet_agent.py

server/internet_agent.py

@@ -1,259 +0,0 @@
-"""Бесплатный интернет-агент с минимум 3 способами поиска"""
-
-import re
-import requests
-import hashlib
-from datetime import datetime, timedelta
-from typing import Dict, List, Tuple, Any, Optional
-from collections import Counter
-
-class FreeInternetAgent:
-    """Интернет-агент с бесплатными поисковыми системами (минимум 3 способа)"""
-    
-    def __init__(self, cache_ttl: int = 3600):
-        self.session = requests.Session()
-        self.session.headers.update({
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-        })
-        self.cache: Dict[str, Tuple[Any, datetime]] = {}
-        self.cache_ttl = cache_ttl
-        self._has_bs4 = False
-        try:
-            import bs4
-            self._has_bs4 = True
-        except ImportError:
-            pass
-        
-        # SearXNG инстансы
-        self.searxng_instances = [
-            "https://searx.be",
-            "https://search.bus-hit.me",
-            "https://searx.nixnet.xyz",
-            "https://searx.tuxcloud.net",
-            "https://searx.moe",
-        ]
-    
-    def _get_cache_key(self, *args, **kwargs) -> str:
-        key = f"{args}_{sorted(kwargs.items())}"
-        return hashlib.md5(key.encode()).hexdigest()
-    
-    def _get_from_cache(self, key: str) -> Optional[Any]:
-        if key in self.cache:
-            data, timestamp = self.cache[key]
-            if datetime.now() - timestamp < timedelta(seconds=self.cache_ttl):
-                return data
-            else:
-                del self.cache[key]
-        return None
-    
-    def _save_to_cache(self, key: str, data: Any) -> None:
-        self.cache[key] = (data, datetime.now())
-    
-    def search_web(self, query: str, num_results: int = 5) -> List[Dict[str, str]]:
-        """Умный поиск с несколькими источниками (минимум 3 способа)"""
-        cache_key = self._get_cache_key('search', query, num_results)
-        cached = self._get_from_cache(cache_key)
-        if cached is not None:
-            return cached
-        
-        results = []
-        
-        # Способ 1: SearXNG (мета-поиск)
-        results = self._search_searxng(query, num_results)
-        
-        # Способ 2: DuckDuckGo API
-        if not results:
-            results = self._search_duckduckgo(query, num_results)
-        
-        # Способ 3: Google (парсинг)
-        if not results:
-            results = self._search_google(query, num_results)
-        
-        # Способ 4: Яндекс (для русского) — опционально
-        if not results and any(ord(c) > 1024 for c in query):
-            results = self._search_yandex(query, num_results)
-        
-        self._save_to_cache(cache_key, results)
-        return results
-    
-    def _search_searxng(self, query: str, num_results: int) -> List[Dict[str, str]]:
-        results = []
-        for instance in self.searxng_instances:
-            try:
-                url = f"{instance}/search"
-                params = {
-                    "q": query,
-                    "format": "json",
-                    "categories": "general",
-                    "engines": "google,bing,duckduckgo,startpage",
-                    "language": "en",
-                    "pageno": 1
-                }
-                response = self.session.get(url, params=params, timeout=20)
-                response.raise_for_status()
-                data = response.json()
-                
-                if 'results' in data:
-                    for item in data['results'][:num_results]:
-                        results.append({
-                            'title': item.get('title', '')[:100],
-                            'url': item.get('url', ''),
-                            'snippet': item.get('content', '')[:200],
-                            'source': 'searxng',
-                            'engine': item.get('engine', '')
-                        })
-                    if results:
-                        print(f"🔍 SearXNG: {len(results)} результатов")
-                        break
-            except Exception as e:
-                continue
-        return results
-    
-    def _search_duckduckgo(self, query: str, num_results: int) -> List[Dict[str, str]]:
-        results = []
-        try:
-            url = f"https://api.duckduckgo.com/?q={query}&format=json&no_html=1&skip_disambig=1"
-            response = self.session.get(url, timeout=15)
-            data = response.json()
-            
-            if 'RelatedTopics' in data:
-                for item in data['RelatedTopics'][:num_results]:
-                    if 'Text' in item and 'FirstURL' in item:
-                        results.append({
-                            'title': item['Text'][:100],
-                            'url': item['FirstURL'],
-                            'snippet': item.get('Text', '')[:200],
-                            'source': 'duckduckgo'
-                        })
-            print(f"🦆 DuckDuckGo: {len(results)} результатов")
-        except Exception as e:
-            print(f"⚠️ DuckDuckGo ошибка: {e}")
-        return results
-    
-    def _search_google(self, query: str, num_results: int) -> List[Dict[str, str]]:
-        results = []
-        if not self._has_bs4:
-            return results
-        try:
-            from bs4 import BeautifulSoup
-            url = f"https://www.google.com/search?q={query}&num={num_results * 2}"
-            response = self.session.get(url, timeout=20)
-            soup = BeautifulSoup(response.text, 'html.parser')
-            
-            for g in soup.find_all('div', class_='g'):
-                title_elem = g.find('h3')
-                link_elem = g.find('a')
-                snippet_elem = g.find('div', class_='VwiC3b')
-                
-                if title_elem and link_elem:
-                    title = title_elem.get_text()
-                    link = link_elem.get('href', '')
-                    snippet = snippet_elem.get_text() if snippet_elem else ''
-                    if link.startswith('/url?q='):
-                        link = link.split('/url?q=')[1].split('&')[0]
-                    if link.startswith('http'):
-                        results.append({
-                            'title': title[:100],
-                            'url': link,
-                            'snippet': snippet[:200],
-                            'source': 'google'
-                        })
-                    if len(results) >= num_results:
-                        break
-            print(f"🔍 Google: {len(results)} результатов")
-        except Exception as e:
-            print(f"⚠️ Google ошибка: {e}")
-        return results
-    
-    def _search_yandex(self, query: str, num_results: int) -> List[Dict[str, str]]:
-        results = []
-        if not self._has_bs4:
-            return results
-        try:
-            from bs4 import BeautifulSoup
-            url = f"https://yandex.ru/search/?text={query}&numdoc={num_results}"
-            response = self.session.get(url, timeout=20)
-            soup = BeautifulSoup(response.text, 'html.parser')
-            
-            for item in soup.find_all('li', class_='serp-item'):
-                link_elem = item.find('a', class_='link')
-                snippet_elem = item.find('div', class_='text-container')
-                if link_elem:
-                    title = link_elem.get_text()
-                    link = link_elem.get('href', '')
-                    snippet = snippet_elem.get_text() if snippet_elem else ''
-                    if link.startswith('http'):
-                        results.append({
-                            'title': title[:100],
-                            'url': link,
-                            'snippet': snippet[:200],
-                            'source': 'yandex'
-                        })
-                    if len(results) >= num_results:
-                        break
-            print(f"🔍 Яндекс: {len(results)} результатов")
-        except Exception as e:
-            print(f"⚠️ Яндекс ошибка: {e}")
-        return results
-    
-    def fetch_page(self, url: str, max_size: int = 50000) -> Dict[str, Any]:
-        try:
-            response = self.session.get(url, timeout=30)
-            response.raise_for_status()
-            content = response.text[:max_size]
-            return {
-                "url": url,
-                "status": response.status_code,
-                "content": content,
-                "length": len(content),
-                "headers": dict(response.headers)
-            }
-        except Exception as e:
-            return {"url": url, "error": str(e)}
-    
-    def analyze_website(self, url: str) -> Dict[str, Any]:
-        page = self.fetch_page(url)
-        if "error" in page:
-            return page
-        content = page.get("content", "")
-        return {
-            "url": url,
-            "title": content.split("<title>")[1].split("</title>")[0] if "<title>" in content else "N/A",
-            "has_forms": "form" in content.lower(),
-            "has_scripts": "<script" in content.lower(),
-            "links_count": content.count("<a "),
-            "size": len(content)
-        }
-    
-    def fetch_multiple(self, urls: List[str], max_total: int = 50000) -> List[Dict[str, Any]]:
-        results = []
-        total = 0
-        for url in urls:
-            page = self.fetch_page(url, max_size=min(10000, max_total - total))
-            if "content" in page:
-                total += len(page["content"])
-            results.append(page)
-            if total >= max_total:
-                break
-        return results
-    
-    def clear_cache(self) -> str:
-        count = len(self.cache)
-        self.cache.clear()
-        return f"🗑️ Кэш очищен ({count} записей)"
-    
-    def get_cache_stats(self) -> str:
-        return f"📊 Кэш: {len(self.cache)} записей, TTL: {self.cache_ttl} секунд"
-    
-    def _extract_keywords(self, content: str) -> List[str]:
-        words = re.findall(r'\b[a-zA-Z]{4,}\b', content.lower())
-        return [w for w, _ in Counter(words).most_common(10)]
-    
-    def _detect_language(self, content: str) -> str:
-        ru_chars = len(re.findall(r'[а-яА-Я]', content))
-        en_chars = len(re.findall(r'[a-zA-Z]', content))
-        if ru_chars > en_chars * 0.5:
-            return "ru"
-        return "en"
-
-INTERNET_AGENT = FreeInternetAgent(cache_ttl=3600)