Update app.py

app.py

@@ -1,7 +1,7 @@
 """
-B2B Content Strategy Planner (Master Industry Edition)
+B2B Content Strategy Planner (Final Robust Edition)
 Deployed for Hugging Face Spaces.
-Features: 25+ Industry Packs, Multi-Model AI, Visual Competitive Intelligence.
+Features: Browser-Like Fetching, Multi-Model AI, Visual Intelligence, 25+ Industry Packs.
 """
 
 import gradio as gr
@@ -21,6 +21,15 @@ from typing import List, Dict, Set, Tuple, Optional
 # 1. CONFIGURATION & MASSIVE TOPIC PACKS
 # ============================================================================
 
+# Global Headers to mimic a real browser (Fixes "Not able to fetch" issues)
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1'
+}
+
 TOPIC_PACKS = {
     # --- Tech & Infra ---
     "cloud": ["cloud", "cloud-computing", "saas", "paas", "iaas", "cloud-migration", "cloud-native", "multicloud", "hybrid-cloud", "serverless"],
@@ -73,7 +82,7 @@ AI_MODELS = [
 ]
 
 # ============================================================================
-# 2. CRAWLER & UTILS
+# 2. ROBUST CRAWLER & FETCHER (FIXED)
 # ============================================================================
 
 def normalize_str(s: str) -> str:
@@ -94,49 +103,96 @@ def classify_stage(url: str) -> str:
     return 'TOFU'
 
 def crawl_website_unlimited(start_url: str):
-    """Robust Crawler"""
+    """Robust Crawler with Real Browser Headers"""
     if not start_url.startswith(('http://', 'https://')): start_url = f'https://{start_url}'
-    domain = urlparse(start_url).netloc.replace('www.', '')
+    
+    # Parse domain safely
+    try:
+        parsed_url = urlparse(start_url)
+        domain = parsed_url.netloc.replace('www.', '')
+        if not domain: return []
+    except: return []
     
     queue = [start_url]
     visited = set()
     found_urls = []
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'}
     
-    while queue and len(found_urls) < 300: # Limit for demo speed
+    # Safety limit
+    max_pages = 250
+    
+    while queue and len(found_urls) < max_pages:
         curr = queue.pop(0)
         if curr in visited: continue
         visited.add(curr)
         
         try:
-            time.sleep(0.05)
-            r = requests.get(curr, headers=headers, timeout=5)
-            if 'text/html' not in r.headers.get('Content-Type', '').lower(): continue
+            time.sleep(0.1) # Small delay to avoid blocking
+            r = requests.get(curr, headers=HEADERS, timeout=8)
             
-            soup = BeautifulSoup(r.text, 'lxml')
-            if r.status_code == 200: found_urls.append(curr)
+            # Skip non-HTML
+            content_type = r.headers.get('Content-Type', '').lower()
+            if 'text/html' not in content_type: continue
+            
+            if r.status_code == 200: 
+                found_urls.append(curr)
+            
+            soup = BeautifulSoup(r.text, 'lxml') # Uses lxml for speed
             
             for a in soup.find_all('a', href=True):
-                full = urljoin(curr, a['href']).split('#')[0].split('?')[0].rstrip('/')
-                if domain in full and full not in visited:
-                    queue.append(full)
-        except: continue
+                href = a['href'].strip()
+                full = urljoin(curr, href)
+                
+                # Basic cleaning
+                clean_url = full.split('#')[0].split('?')[0].rstrip('/')
+                
+                # Check domain match
+                if domain in urlparse(clean_url).netloc:
+                    # Filter junk
+                    if not any(ext in clean_url.lower() for ext in ['.pdf', '.jpg', '.png', '.css', '.js', 'login', 'signup']):
+                        if clean_url not in visited and clean_url not in queue:
+                            queue.append(clean_url)
+        except Exception: 
+            continue
     
     return list(set(found_urls))
 
 def fetch_sitemap_links(url: str):
-    # Try XML first
+    """Tries multiple sitemap locations, falls back to crawl"""
+    url = url.strip()
     if not url.startswith('http'): url = f'https://{url}'
-    try:
-        r = requests.get(f"{url.rstrip('/')}/sitemap.xml", timeout=5)
-        if r.status_code == 200:
-            urls = re.findall(r'<loc>(.*?)</loc>', r.text)
-            if len(urls) > 5: return list(set(urls)), f"✅ Found {len(urls)} URLs (XML)"
-    except: pass
     
-    # Fallback crawl
-    urls = crawl_website_unlimited(url)
-    return urls, f"✅ Crawled {len(urls)} pages"
+    base_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}"
+    
+    # Common sitemap locations
+    candidates = [
+        url,
+        f"{base_url}/sitemap.xml",
+        f"{base_url}/sitemap_index.xml",
+        f"{base_url}/wp-sitemap.xml",
+        f"{base_url}/page-sitemap.xml"
+    ]
+    
+    found_urls = []
+    
+    # 1. Try Sitemaps
+    for candidate in candidates:
+        if not candidate.endswith('.xml') and candidate != url: continue
+        try:
+            r = requests.get(candidate, headers=HEADERS, timeout=10)
+            if r.status_code == 200:
+                # Use regex for speed & robustness against bad XML
+                urls = re.findall(r'<loc>(.*?)</loc>', r.text)
+                if len(urls) > 5:
+                    found_urls = list(set(urls))
+                    return found_urls, f"✅ Found {len(found_urls)} URLs via XML Sitemap ({candidate})"
+        except: continue
+            
+    # 2. Fallback to Crawler if sitemap failed
+    crawl_urls = crawl_website_unlimited(base_url)
+    if len(crawl_urls) > 0:
+        return crawl_urls, f"✅ Crawled {len(crawl_urls)} pages (Sitemap not found)"
+        
+    return [], "❌ Could not fetch content. Site might be blocking bots."
 
 def categorize_links(urls):
     cats = defaultdict(list)
@@ -150,7 +206,13 @@ def categorize_links(urls):
                 break
         if not assigned:
             parts = list(filter(None, urlparse(url).path.split('/')))
-            if parts: cats[parts[0].title()].append(parts[-1])
+            if parts: 
+                # Use first folder as category if sensible
+                cat_candidate = parts[0].title()
+                if len(cat_candidate) < 20 and not any(x in cat_candidate.lower() for x in ['202', '0', '1']):
+                    cats[cat_candidate].append(parts[-1])
+                else:
+                    cats["General"].append(parts[-1])
     return dict(cats)
 
 # ============================================================================
@@ -379,7 +441,7 @@ def ui_fetch(url):
     cats = categorize_links(urls)
     state['urls'] = urls
     cat_list = ["All"] + sorted(list(cats.keys()))
-    return urls, cats, msg, gr.update(choices=cat_list, value="All"), gr.update(choices=[])
+    return urls, cats, msg, gr.update(choices=cat_list, value="All"), gr.update(choices=[], value=None)
 
 def ui_update_topics(category, categories_data):
     if not categories_data: return gr.update(choices=[])
@@ -392,7 +454,7 @@ def ui_analyze_planner(urls, category, topic):
     if not urls: return "Fetch data first.", None, None
     if not topic: return "Select a topic.", None, None
     
-    # Match logic reused from previous (simplified for brevity)
+    # Match logic
     matched = [u for u in urls if normalize_str(topic) in path_norm(u)]
     
     stages = [classify_stage(u) for u in matched]
@@ -417,14 +479,6 @@ def ui_add_comp(url, current_comps):
     txt = "\n".join([f"✅ {c['domain']}: {c['stats']['Count']} pages" for c in current_comps])
     return current_comps, txt, ""
 
-def ui_comp_search(user_urls, competitors, gap_topic, str_topic):
-    if not user_urls: return "Fetch user data first.", ""
-    
-    g_html, _ = analyze_gaps_strengths(user_urls, competitors, gap_topic)
-    _, s_html = analyze_gaps_strengths(user_urls, competitors, str_topic)
-    
-    return g_html, s_html
-
 def ui_comp_master(user_urls, competitors):
     if not user_urls: return None, "", "", ""
     
@@ -444,7 +498,6 @@ def ui_comp_master(user_urls, competitors):
     g_html, s_html = analyze_gaps_strengths(user_urls, competitors, None)
     
     # 3. Strategy
-    # Extract raw lists for summary
     raw_gaps = g_html.count("<span") 
     raw_str = s_html.count("<span")
     strat = generate_strategic_summary(range(raw_gaps), range(raw_str))