Update app.py

app.py

@@ -1,6 +1,6 @@
 """
-B2B Content Strategy Planner (Hybrid Edition)
-Combines your Working Smart Crawler with Advanced Analytics (Radar Charts, AI Scoring, Gap Analysis).
+B2B Content Strategy Planner (Final Fixed Edition)
+Deployed for Hugging Face Spaces.
 """
 
 import gradio as gr
@@ -16,7 +16,7 @@ import plotly.express as px
 from typing import List, Dict, Set, Tuple, Optional
 
 # ============================================================================
-# 1. CONFIGURATION & TOPIC PACKS (FULL)
+# 1. CONFIGURATION & TOPIC PACKS
 # ============================================================================
 
 TOPIC_PACKS = {
@@ -47,7 +47,7 @@ CATEGORY_HINTS = {
 }
 
 # ============================================================================
-# 2. YOUR WORKING CRAWLER LOGIC (PRESERVED)
+# 2. CRAWLER LOGIC (Smart & Deep)
 # ============================================================================
 
 def normalize_str(s: str) -> str:
@@ -63,14 +63,8 @@ def path_norm(url: str) -> str:
     except:
         return normalize_str(url)
 
-def slugify_topic(topic_text: str) -> str:
-    return normalize_str(topic_text)
-
 def crawl_and_generate_sitemap(start_url: str, max_pages: int = 150) -> Tuple[List[str], str]:
-    """
-    Your Crawler Logic: Crawls a website to generate a sitemap dynamically.
-    Increased limit to 150 for better analysis.
-    """
+    """Crawls a website to generate a sitemap dynamically."""
     if not start_url.startswith(('http://', 'https://')):
         start_url = f'https://{start_url}'
     
@@ -90,8 +84,8 @@ def crawl_and_generate_sitemap(start_url: str, max_pages: int = 150) -> Tuple[Li
     while queue and len(found_urls) < max_pages:
         current_url = queue.pop(0)
         try:
-            time.sleep(0.1) # Respectful delay
-            response = requests.get(current_url, headers=headers, timeout=8)
+            time.sleep(0.05) # Polite delay
+            response = requests.get(current_url, headers=headers, timeout=6)
             
             content_type = response.headers.get('Content-Type', '').lower()
             if 'text/html' not in content_type:
@@ -112,7 +106,7 @@ def crawl_and_generate_sitemap(start_url: str, max_pages: int = 150) -> Tuple[Li
                 # Logic: Internal Domain, No Files, No System Pages
                 is_internal = parsed_url.netloc == domain or parsed_url.netloc == ''
                 is_file = any(clean_url.endswith(ext) for ext in ['.pdf', '.jpg', '.png', '.zip', '.css', '.js'])
-                is_ignored = any(x in clean_url.lower() for x in ['/login', '/cart', '/checkout', '/account', 'javascript:', 'mailto:'])
+                is_ignored = any(x in clean_url.lower() for x in ['/login', '/cart', '/checkout', 'javascript:', 'mailto:', 'tel:'])
 
                 if is_internal and not is_file and not is_ignored:
                     if clean_url not in visited:
@@ -185,7 +179,7 @@ def fetch_sitemap_links(user_input: str) -> Tuple[List[str], str]:
     return crawl_and_generate_sitemap(base_url)
 
 # ============================================================================
-# 3. ADVANCED ANALYSIS (Merged from my code)
+# 3. ANALYSIS & CATEGORIZATION
 # ============================================================================
 
 def classify_stage(url: str) -> str:
@@ -201,14 +195,11 @@ def expand_topic_variants(topic_slug: str) -> Set[str]:
     variants.add(topic_slug.replace('-', ''))
     variants.add(topic_slug.replace('-', ' '))
     
-    # Add pack keywords
     for pack_name, keywords in TOPIC_PACKS.items():
         if topic_slug in [normalize_str(k) for k in keywords] or topic_slug == pack_name:
             for kw in keywords:
                 n_kw = normalize_str(kw)
-                variants.add(n_kw)
-                variants.add(n_kw.replace('-', ''))
-                variants.add(n_kw.replace('-', ' '))
+                variants.update([n_kw, n_kw.replace('-', ''), n_kw.replace('-', ' ')])
             break
     return variants
 
@@ -217,6 +208,7 @@ def categorize_links(urls: List[str]) -> Dict[str, List[str]]:
     for url in urls:
         norm = path_norm(url)
         assigned = False
+        
         # 1. Check Hints
         for cat_name, hints in CATEGORY_HINTS.items():
             if any(h in norm for h in hints):
@@ -225,16 +217,25 @@ def categorize_links(urls: List[str]) -> Dict[str, List[str]]:
                 categories[cat_name].append(topic)
                 assigned = True
                 break
-        # 2. Check Folder Structure
+        
+        # 2. Check Folder Structure (Fallback)
         if not assigned:
             parts = list(filter(None, urlparse(url).path.split('/')))
             if len(parts) > 0:
-                categories[parts[0].title()].append(parts[-1])
+                cat = parts[0].title()
+                # Avoid junk categories (dates, short codes)
+                if len(cat) > 2 and not cat.isdigit():
+                    categories[cat].append(parts[-1])
+            else:
+                # 3. Root Level / Uncategorized
+                categories["General"].append(norm)
+
     return dict(categories)
 
 def match_topic_urls(links, topic, category, match_mode, synonyms="", neg_keywords=""):
     # Filter Category
     if category and category != "All":
+        # Only include if URL belongs to selected category
         filtered = [u for u in links if category in categorize_links([u])]
     else:
         filtered = links
@@ -271,11 +272,9 @@ def match_topic_urls(links, topic, category, match_mode, synonyms="", neg_keywor
     return [x[0] for x in results]
 
 def score_ai_output(output_text: str, topic: str, stage: str) -> dict:
-    """7-Point Scoring System"""
     scores = {'total': 0}
     lower_out = output_text.lower()
     
-    # Scoring Logic
     matches = sum(lower_out.count(w) for w in normalize_str(topic).split('-'))
     scores['Relevance'] = min(20, matches * 2)
     
@@ -303,14 +302,12 @@ def generate_radar(user_urls, competitors):
     categories = ['TOFU', 'MOFU', 'BOFU']
     fig = go.Figure()
     
-    # User Data
     if user_urls:
         stages = [classify_stage(u) for u in user_urls]
         total = len(stages) or 1
         stats = [(stages.count(s)/total)*100 for s in categories]
         fig.add_trace(go.Scatterpolar(r=stats, theta=categories, fill='toself', name='You', line_color='#3b82f6'))
     
-    # Competitor Data
     colors = ['#ef4444', '#10b981', '#f59e0b', '#8b5cf6']
     for i, c in enumerate(competitors):
         s = c['stats']
@@ -346,14 +343,12 @@ def analyze_gaps(user_urls, competitors, filter_topic):
 # 5. UI EVENT HANDLERS
 # ============================================================================
 
-state = {'urls': [], 'categories': {}, 'competitors': []}
-
 def ui_fetch_data(url):
     urls, msg = fetch_sitemap_links(url)
     cats = categorize_links(urls)
-    state['urls'] = urls
-    state['categories'] = cats
-    return urls, cats, msg, list(cats.keys())
+    # KEY FIX: Return gr.update for the Dropdown
+    cat_choices = ["All"] + list(cats.keys())
+    return urls, cats, msg, gr.update(choices=cat_choices, value="All")
 
 def ui_analyze_planner(urls, category, topic, mode, syns, negs):
     if not urls: return "Fetch data first.", None, None
@@ -375,28 +370,20 @@ def ui_analyze_planner(urls, category, topic, mode, syns, negs):
     return summary, matched, None
 
 def ui_generate_ideas(topic, stage):
-    # Mock AI Generation (Replace with actual API key in production)
     mock_res = f"""# 🚀 {stage} Content Ideas for {topic}
-
 ## 1. The Ultimate Guide to {topic}
 **Format:** Blog Post
 **Why:** Captures high-volume search traffic.
-**Structure:** Definition, Benefits, Trends.
-
 ## 2. {topic} Maturity Model
 **Format:** Whitepaper
 **Why:** Helps leads evaluate their current state.
-
-## 3. Case Study: Solving {topic} at Scale
-**Format:** Success Story
-**Why:** Social proof for decision makers.
     """
     scores = score_ai_output(mock_res, topic, stage)
     score_txt = f"### 🏆 AI Score: {scores['total']}/100\n" + "\n".join(f"- {k}: {v}" for k,v in scores.items() if k!='total')
     return mock_res, score_txt
 
 def ui_add_competitor(url, current_comps):
-    urls, msg = fetch_sitemap_links(url) # Uses same hybrid crawler
+    urls, msg = fetch_sitemap_links(url)
     domain = urlparse(url if url.startswith('http') else f'https://{url}').netloc
     
     stages = [classify_stage(u) for u in urls]
@@ -441,7 +428,8 @@ with gr.Blocks(theme=gr.themes.Soft(), title="Content Strategy Master") as demo:
             fetch_log = gr.Textbox(label="Status", lines=1)
             
             with gr.Row():
-                cat_dd = gr.Dropdown(label="Category", choices=[])
+                # FIXED: Dropdown is now updated properly via gr.update
+                cat_dd = gr.Dropdown(label="Category", choices=[], allow_custom_value=True)
                 topic_in = gr.Textbox(label="Topic")
             
             with gr.Accordion("Advanced Matching", open=False):
@@ -485,13 +473,4 @@ with gr.Blocks(theme=gr.themes.Soft(), title="Content Strategy Master") as demo:
     
     analyze_btn.click(ui_analyze_planner, 
         [state_urls_ui, cat_dd, topic_in, match_mode, synonyms, neg_kw], 
-        [plan_summary, plan_urls, gr.State()])
-        
-    gen_btn.click(ui_generate_ideas, [topic_in, gen_stage], [idea_out, score_out])
-    
-    add_comp.click(ui_add_competitor, [comp_url, state_comps_ui], [state_comps_ui, comp_display, comp_url])
-    
-    comp_analyze.click(ui_analyze_comp_full, [state_urls_ui, state_comps_ui, gap_filter], [radar_plot, gap_res])
-
-if __name__ == "__main__":
-    demo.launch()
+        [plan_summary, plan_urls, gr.State()])