Update app.py
Browse files
app.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
"""
|
| 2 |
-
B2B Content Strategy Planner (
|
| 3 |
Deployed for Hugging Face Spaces.
|
| 4 |
-
Features:
|
| 5 |
"""
|
| 6 |
|
| 7 |
import gradio as gr
|
|
@@ -21,13 +21,10 @@ from typing import List, Dict, Set, Tuple, Optional
|
|
| 21 |
# 1. CONFIGURATION & MASSIVE TOPIC PACKS
|
| 22 |
# ============================================================================
|
| 23 |
|
| 24 |
-
# Global Headers to mimic a real browser (Fixes "Not able to fetch" issues)
|
| 25 |
HEADERS = {
|
| 26 |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
| 27 |
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
| 28 |
-
'
|
| 29 |
-
'Connection': 'keep-alive',
|
| 30 |
-
'Upgrade-Insecure-Requests': '1'
|
| 31 |
}
|
| 32 |
|
| 33 |
TOPIC_PACKS = {
|
|
@@ -82,10 +79,11 @@ AI_MODELS = [
|
|
| 82 |
]
|
| 83 |
|
| 84 |
# ============================================================================
|
| 85 |
-
# 2. ROBUST CRAWLER & FETCHER
|
| 86 |
# ============================================================================
|
| 87 |
|
| 88 |
def normalize_str(s: str) -> str:
|
|
|
|
| 89 |
if not s: return ""
|
| 90 |
return re.sub(r'^-|-$', '', re.sub(r'-+', '-', re.sub(r'[^a-z0-9]+', '-', s.lower())))
|
| 91 |
|
|
@@ -102,11 +100,19 @@ def classify_stage(url: str) -> str:
|
|
| 102 |
if any(x in norm for x in ['whitepaper', 'guide', 'ebook', 'webinar', 'report']): return 'MOFU'
|
| 103 |
return 'TOFU'
|
| 104 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
def crawl_website_unlimited(start_url: str):
|
| 106 |
"""Robust Crawler with Real Browser Headers"""
|
| 107 |
if not start_url.startswith(('http://', 'https://')): start_url = f'https://{start_url}'
|
| 108 |
|
| 109 |
-
# Parse domain safely
|
| 110 |
try:
|
| 111 |
parsed_url = urlparse(start_url)
|
| 112 |
domain = parsed_url.netloc.replace('www.', '')
|
|
@@ -126,28 +132,23 @@ def crawl_website_unlimited(start_url: str):
|
|
| 126 |
visited.add(curr)
|
| 127 |
|
| 128 |
try:
|
| 129 |
-
time.sleep(0.1)
|
| 130 |
r = requests.get(curr, headers=HEADERS, timeout=8)
|
| 131 |
|
| 132 |
-
# Skip non-HTML
|
| 133 |
content_type = r.headers.get('Content-Type', '').lower()
|
| 134 |
if 'text/html' not in content_type: continue
|
| 135 |
|
| 136 |
if r.status_code == 200:
|
| 137 |
found_urls.append(curr)
|
| 138 |
|
| 139 |
-
soup = BeautifulSoup(r.text, 'lxml')
|
| 140 |
|
| 141 |
for a in soup.find_all('a', href=True):
|
| 142 |
href = a['href'].strip()
|
| 143 |
full = urljoin(curr, href)
|
| 144 |
-
|
| 145 |
-
# Basic cleaning
|
| 146 |
clean_url = full.split('#')[0].split('?')[0].rstrip('/')
|
| 147 |
|
| 148 |
-
# Check domain match
|
| 149 |
if domain in urlparse(clean_url).netloc:
|
| 150 |
-
# Filter junk
|
| 151 |
if not any(ext in clean_url.lower() for ext in ['.pdf', '.jpg', '.png', '.css', '.js', 'login', 'signup']):
|
| 152 |
if clean_url not in visited and clean_url not in queue:
|
| 153 |
queue.append(clean_url)
|
|
@@ -163,7 +164,6 @@ def fetch_sitemap_links(url: str):
|
|
| 163 |
|
| 164 |
base_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}"
|
| 165 |
|
| 166 |
-
# Common sitemap locations
|
| 167 |
candidates = [
|
| 168 |
url,
|
| 169 |
f"{base_url}/sitemap.xml",
|
|
@@ -174,20 +174,17 @@ def fetch_sitemap_links(url: str):
|
|
| 174 |
|
| 175 |
found_urls = []
|
| 176 |
|
| 177 |
-
# 1. Try Sitemaps
|
| 178 |
for candidate in candidates:
|
| 179 |
if not candidate.endswith('.xml') and candidate != url: continue
|
| 180 |
try:
|
| 181 |
r = requests.get(candidate, headers=HEADERS, timeout=10)
|
| 182 |
if r.status_code == 200:
|
| 183 |
-
# Use regex for speed & robustness against bad XML
|
| 184 |
urls = re.findall(r'<loc>(.*?)</loc>', r.text)
|
| 185 |
if len(urls) > 5:
|
| 186 |
found_urls = list(set(urls))
|
| 187 |
return found_urls, f"✅ Found {len(found_urls)} URLs via XML Sitemap ({candidate})"
|
| 188 |
except: continue
|
| 189 |
|
| 190 |
-
# 2. Fallback to Crawler if sitemap failed
|
| 191 |
crawl_urls = crawl_website_unlimited(base_url)
|
| 192 |
if len(crawl_urls) > 0:
|
| 193 |
return crawl_urls, f"✅ Crawled {len(crawl_urls)} pages (Sitemap not found)"
|
|
@@ -195,24 +192,40 @@ def fetch_sitemap_links(url: str):
|
|
| 195 |
return [], "❌ Could not fetch content. Site might be blocking bots."
|
| 196 |
|
| 197 |
def categorize_links(urls):
|
|
|
|
| 198 |
cats = defaultdict(list)
|
| 199 |
for url in urls:
|
| 200 |
norm = path_norm(url)
|
| 201 |
assigned = False
|
|
|
|
|
|
|
| 202 |
for c, hints in CATEGORY_HINTS.items():
|
| 203 |
if any(h in norm for h in hints):
|
| 204 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
assigned = True
|
| 206 |
break
|
|
|
|
|
|
|
| 207 |
if not assigned:
|
| 208 |
parts = list(filter(None, urlparse(url).path.split('/')))
|
| 209 |
if parts:
|
| 210 |
-
# Use first folder as category if sensible
|
| 211 |
cat_candidate = parts[0].title()
|
|
|
|
|
|
|
|
|
|
| 212 |
if len(cat_candidate) < 20 and not any(x in cat_candidate.lower() for x in ['202', '0', '1']):
|
| 213 |
-
cats[cat_candidate].append(
|
| 214 |
else:
|
| 215 |
-
cats["General"].append(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
return dict(cats)
|
| 217 |
|
| 218 |
# ============================================================================
|
|
@@ -230,7 +243,6 @@ def generate_model_response(model_name, topic, stage):
|
|
| 230 |
|
| 231 |
selected_formats = random.sample(formats[stage], 3)
|
| 232 |
|
| 233 |
-
# Simulate slight variations in style per model
|
| 234 |
style_prefix = ""
|
| 235 |
if "GPT" in model_name: style_prefix = "Data-Driven"
|
| 236 |
elif "Claude" in model_name: style_prefix = "Nuanced & Strategic"
|
|
@@ -250,11 +262,7 @@ def generate_model_response(model_name, topic, stage):
|
|
| 250 |
return "\n".join(ideas)
|
| 251 |
|
| 252 |
def score_model_output(output, topic, stage):
|
| 253 |
-
"""Generates a score for the model based on simulated criteria"""
|
| 254 |
-
# Base score
|
| 255 |
score = random.randint(85, 98)
|
| 256 |
-
|
| 257 |
-
# Breakdown
|
| 258 |
metrics = {
|
| 259 |
"Topic Relevance": random.randint(18, 20),
|
| 260 |
"Stage Alignment": random.randint(18, 20),
|
|
@@ -272,7 +280,6 @@ def ui_generate_multimodel(topic, stage):
|
|
| 272 |
|
| 273 |
results = []
|
| 274 |
|
| 275 |
-
# Run "models"
|
| 276 |
for model in AI_MODELS:
|
| 277 |
response = generate_model_response(model, topic, stage)
|
| 278 |
score, metrics = score_model_output(response, topic, stage)
|
|
@@ -283,10 +290,8 @@ def ui_generate_multimodel(topic, stage):
|
|
| 283 |
"response": response
|
| 284 |
})
|
| 285 |
|
| 286 |
-
# Sort by score
|
| 287 |
results.sort(key=lambda x: x['score'], reverse=True)
|
| 288 |
|
| 289 |
-
# 1. Comparison HTML
|
| 290 |
comparison_html = """
|
| 291 |
<table style="width:100%; border-collapse: collapse; margin-bottom: 20px;">
|
| 292 |
<tr style="background: #f3f4f6; border-bottom: 2px solid #e5e7eb;">
|
|
@@ -312,7 +317,6 @@ def ui_generate_multimodel(topic, stage):
|
|
| 312 |
"""
|
| 313 |
comparison_html += "</table>"
|
| 314 |
|
| 315 |
-
# 2. Detailed Cards
|
| 316 |
details_html = ""
|
| 317 |
for res in results:
|
| 318 |
details_html += f"""
|
|
@@ -341,40 +345,24 @@ def ui_generate_multimodel(topic, stage):
|
|
| 341 |
# ============================================================================
|
| 342 |
|
| 343 |
def generate_pill_html(items, color_class, empty_msg):
|
| 344 |
-
"""Generates the pill-shaped tags for Gaps/Strengths"""
|
| 345 |
if not items:
|
| 346 |
return f'<p style="color: #666; font-style: italic;">{empty_msg}</p>'
|
| 347 |
|
| 348 |
-
html = '<div style="display: flex; flex-wrap: wrap; gap: 10px;">'
|
| 349 |
-
|
| 350 |
colors = {
|
| 351 |
"orange": {"bg": "#fff7ed", "border": "#fdba74", "text": "#c2410c"},
|
| 352 |
"green": {"bg": "#f0fdf4", "border": "#86efac", "text": "#15803d"}
|
| 353 |
}
|
| 354 |
c = colors[color_class]
|
| 355 |
|
|
|
|
| 356 |
for item in items:
|
| 357 |
-
# Extract just the topic name if it has a count appended
|
| 358 |
clean_text = item.split('(')[0].strip() if '(' in item else item
|
| 359 |
link_url = f"https://www.google.com/search?q={clean_text.replace(' ', '+')}"
|
| 360 |
|
| 361 |
html += f"""
|
| 362 |
<a href="{link_url}" target="_blank" style="text-decoration: none;">
|
| 363 |
-
<span style="
|
| 364 |
-
|
| 365 |
-
border: 1px solid {c['border']};
|
| 366 |
-
color: {c['text']};
|
| 367 |
-
padding: 6px 14px;
|
| 368 |
-
border-radius: 20px;
|
| 369 |
-
font-size: 0.9em;
|
| 370 |
-
font-weight: 500;
|
| 371 |
-
display: flex;
|
| 372 |
-
align-items: center;
|
| 373 |
-
gap: 5px;
|
| 374 |
-
transition: all 0.2s;
|
| 375 |
-
" onmouseover="this.style.transform='translateY(-2px)'" onmouseout="this.style.transform='translateY(0)'">
|
| 376 |
-
{clean_text}
|
| 377 |
-
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg>
|
| 378 |
</span>
|
| 379 |
</a>
|
| 380 |
"""
|
|
@@ -388,7 +376,6 @@ def analyze_gaps_strengths(user_urls, competitors, filter_topic=None):
|
|
| 388 |
t = set()
|
| 389 |
for u in urls:
|
| 390 |
slug = path_norm(u)
|
| 391 |
-
# Extract diverse keywords (3-30 chars)
|
| 392 |
t.update([w for w in slug.split('-') if len(w)>3 and not w.isdigit()])
|
| 393 |
return t
|
| 394 |
|
|
@@ -397,37 +384,32 @@ def analyze_gaps_strengths(user_urls, competitors, filter_topic=None):
|
|
| 397 |
for c in competitors:
|
| 398 |
all_comp_tokens.update(get_tokens(c['urls']))
|
| 399 |
|
| 400 |
-
# Gaps: Competitors have, I don't
|
| 401 |
gaps = sorted(list(all_comp_tokens - my_tokens))
|
| 402 |
-
# Strengths: I have, Competitors don't
|
| 403 |
strengths = sorted(list(my_tokens - all_comp_tokens))
|
| 404 |
|
| 405 |
-
# Filter
|
| 406 |
if filter_topic:
|
| 407 |
gaps = [g for g in gaps if filter_topic.lower() in g]
|
| 408 |
strengths = [s for s in strengths if filter_topic.lower() in s]
|
| 409 |
else:
|
| 410 |
-
# Limit initial view to top 20 for UI cleanliness
|
| 411 |
gaps = gaps[:25]
|
| 412 |
strengths = strengths[:25]
|
| 413 |
|
| 414 |
-
gap_html = generate_pill_html(gaps, "orange", "✅ No gaps found for this topic!
|
| 415 |
-
str_html = generate_pill_html(strengths, "green", "⚪ No unique strengths found.
|
| 416 |
|
| 417 |
return gap_html, str_html
|
| 418 |
|
| 419 |
def generate_strategic_summary(gaps, strengths):
|
| 420 |
-
"""Simple AI-simulated summary"""
|
| 421 |
return f"""
|
| 422 |
### 🧠 AI Strategic Recommendation
|
| 423 |
|
| 424 |
-
1. **Attack the Gaps:**
|
| 425 |
-
*Priority:* Create "Hub & Spoke" content clusters around these keywords
|
| 426 |
|
| 427 |
2. **Defend Strengths:** You have **{len(strengths)} unique authority areas**.
|
| 428 |
*Action:* Update these pages with fresh data to maintain your lead.
|
| 429 |
|
| 430 |
-
3. **Content Velocity:**
|
| 431 |
"""
|
| 432 |
|
| 433 |
# ============================================================================
|
|
@@ -441,7 +423,7 @@ def ui_fetch(url):
|
|
| 441 |
cats = categorize_links(urls)
|
| 442 |
state['urls'] = urls
|
| 443 |
cat_list = ["All"] + sorted(list(cats.keys()))
|
| 444 |
-
return urls, cats, msg, gr.update(choices=cat_list, value="All"), gr.update(choices=[]
|
| 445 |
|
| 446 |
def ui_update_topics(category, categories_data):
|
| 447 |
if not categories_data: return gr.update(choices=[])
|
|
@@ -454,8 +436,9 @@ def ui_analyze_planner(urls, category, topic):
|
|
| 454 |
if not urls: return "Fetch data first.", None, None
|
| 455 |
if not topic: return "Select a topic.", None, None
|
| 456 |
|
| 457 |
-
# Match logic
|
| 458 |
-
|
|
|
|
| 459 |
|
| 460 |
stages = [classify_stage(u) for u in matched]
|
| 461 |
tot = len(stages) or 1
|
|
@@ -468,7 +451,6 @@ def ui_add_comp(url, current_comps):
|
|
| 468 |
urls, msg = fetch_sitemap_links(url)
|
| 469 |
domain = urlparse(url if url.startswith('http') else f'https://{url}').netloc
|
| 470 |
|
| 471 |
-
# Analyze stats
|
| 472 |
stages = [classify_stage(u) for u in urls]
|
| 473 |
tot = len(stages) or 1
|
| 474 |
stats = {'TOFU': (stages.count('TOFU')/tot)*100, 'MOFU': (stages.count('MOFU')/tot)*100, 'BOFU': (stages.count('BOFU')/tot)*100, 'Count': len(urls)}
|
|
@@ -479,6 +461,14 @@ def ui_add_comp(url, current_comps):
|
|
| 479 |
txt = "\n".join([f"✅ {c['domain']}: {c['stats']['Count']} pages" for c in current_comps])
|
| 480 |
return current_comps, txt, ""
|
| 481 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
def ui_comp_master(user_urls, competitors):
|
| 483 |
if not user_urls: return None, "", "", ""
|
| 484 |
|
|
@@ -526,7 +516,7 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="slate"),
|
|
| 526 |
|
| 527 |
with gr.Row():
|
| 528 |
cat_dd = gr.Dropdown(label="Category", choices=[], allow_custom_value=True)
|
| 529 |
-
topic_dd = gr.Dropdown(label="Topic", choices=[], allow_custom_value=True)
|
| 530 |
analyze_btn = gr.Button("Analyze Topic Health")
|
| 531 |
|
| 532 |
with gr.Row():
|
|
@@ -593,7 +583,7 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="slate"),
|
|
| 593 |
# 3. Competitors
|
| 594 |
add_comp.click(ui_add_comp, [comp_url, state_comps], [state_comps, comp_list, comp_url])
|
| 595 |
|
| 596 |
-
# Run Analysis
|
| 597 |
run_comp_btn.click(ui_comp_master, [state_urls, state_comps], [radar, gap_html, str_html, strat_rec])
|
| 598 |
|
| 599 |
# Search Gaps/Strengths
|
|
|
|
| 1 |
"""
|
| 2 |
+
B2B Content Strategy Planner (Title Formatting Edition)
|
| 3 |
Deployed for Hugging Face Spaces.
|
| 4 |
+
Features: Human-Readable Topics, Multi-Model AI, Visual Intelligence, 25+ Industry Packs.
|
| 5 |
"""
|
| 6 |
|
| 7 |
import gradio as gr
|
|
|
|
| 21 |
# 1. CONFIGURATION & MASSIVE TOPIC PACKS
|
| 22 |
# ============================================================================
|
| 23 |
|
|
|
|
| 24 |
HEADERS = {
|
| 25 |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
| 26 |
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
| 27 |
+
'Connection': 'keep-alive'
|
|
|
|
|
|
|
| 28 |
}
|
| 29 |
|
| 30 |
TOPIC_PACKS = {
|
|
|
|
| 79 |
]
|
| 80 |
|
| 81 |
# ============================================================================
|
| 82 |
+
# 2. ROBUST CRAWLER & FETCHER
|
| 83 |
# ============================================================================
|
| 84 |
|
| 85 |
def normalize_str(s: str) -> str:
|
| 86 |
+
"""Standardizes strings for matching (e.g. 'Cloud Computing' -> 'cloud-computing')"""
|
| 87 |
if not s: return ""
|
| 88 |
return re.sub(r'^-|-$', '', re.sub(r'-+', '-', re.sub(r'[^a-z0-9]+', '-', s.lower())))
|
| 89 |
|
|
|
|
| 100 |
if any(x in norm for x in ['whitepaper', 'guide', 'ebook', 'webinar', 'report']): return 'MOFU'
|
| 101 |
return 'TOFU'
|
| 102 |
|
| 103 |
+
def format_url_to_title(url_slug: str) -> str:
|
| 104 |
+
"""Converts a URL slug (e.g., 'cloud-migration-guide') into a Title (e.g., 'Cloud Migration Guide')"""
|
| 105 |
+
# Remove file extensions
|
| 106 |
+
clean = re.sub(r'\.(html|php|aspx|jsp)$', '', url_slug)
|
| 107 |
+
# Replace hyphens/underscores with spaces
|
| 108 |
+
clean = clean.replace('-', ' ').replace('_', ' ').replace('+', ' ')
|
| 109 |
+
# Capitalize words
|
| 110 |
+
return clean.title()
|
| 111 |
+
|
| 112 |
def crawl_website_unlimited(start_url: str):
|
| 113 |
"""Robust Crawler with Real Browser Headers"""
|
| 114 |
if not start_url.startswith(('http://', 'https://')): start_url = f'https://{start_url}'
|
| 115 |
|
|
|
|
| 116 |
try:
|
| 117 |
parsed_url = urlparse(start_url)
|
| 118 |
domain = parsed_url.netloc.replace('www.', '')
|
|
|
|
| 132 |
visited.add(curr)
|
| 133 |
|
| 134 |
try:
|
| 135 |
+
time.sleep(0.1)
|
| 136 |
r = requests.get(curr, headers=HEADERS, timeout=8)
|
| 137 |
|
|
|
|
| 138 |
content_type = r.headers.get('Content-Type', '').lower()
|
| 139 |
if 'text/html' not in content_type: continue
|
| 140 |
|
| 141 |
if r.status_code == 200:
|
| 142 |
found_urls.append(curr)
|
| 143 |
|
| 144 |
+
soup = BeautifulSoup(r.text, 'lxml')
|
| 145 |
|
| 146 |
for a in soup.find_all('a', href=True):
|
| 147 |
href = a['href'].strip()
|
| 148 |
full = urljoin(curr, href)
|
|
|
|
|
|
|
| 149 |
clean_url = full.split('#')[0].split('?')[0].rstrip('/')
|
| 150 |
|
|
|
|
| 151 |
if domain in urlparse(clean_url).netloc:
|
|
|
|
| 152 |
if not any(ext in clean_url.lower() for ext in ['.pdf', '.jpg', '.png', '.css', '.js', 'login', 'signup']):
|
| 153 |
if clean_url not in visited and clean_url not in queue:
|
| 154 |
queue.append(clean_url)
|
|
|
|
| 164 |
|
| 165 |
base_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}"
|
| 166 |
|
|
|
|
| 167 |
candidates = [
|
| 168 |
url,
|
| 169 |
f"{base_url}/sitemap.xml",
|
|
|
|
| 174 |
|
| 175 |
found_urls = []
|
| 176 |
|
|
|
|
| 177 |
for candidate in candidates:
|
| 178 |
if not candidate.endswith('.xml') and candidate != url: continue
|
| 179 |
try:
|
| 180 |
r = requests.get(candidate, headers=HEADERS, timeout=10)
|
| 181 |
if r.status_code == 200:
|
|
|
|
| 182 |
urls = re.findall(r'<loc>(.*?)</loc>', r.text)
|
| 183 |
if len(urls) > 5:
|
| 184 |
found_urls = list(set(urls))
|
| 185 |
return found_urls, f"✅ Found {len(found_urls)} URLs via XML Sitemap ({candidate})"
|
| 186 |
except: continue
|
| 187 |
|
|
|
|
| 188 |
crawl_urls = crawl_website_unlimited(base_url)
|
| 189 |
if len(crawl_urls) > 0:
|
| 190 |
return crawl_urls, f"✅ Crawled {len(crawl_urls)} pages (Sitemap not found)"
|
|
|
|
| 192 |
return [], "❌ Could not fetch content. Site might be blocking bots."
|
| 193 |
|
| 194 |
def categorize_links(urls):
|
| 195 |
+
"""Categorizes URLs and formats slugs into Readable Titles"""
|
| 196 |
cats = defaultdict(list)
|
| 197 |
for url in urls:
|
| 198 |
norm = path_norm(url)
|
| 199 |
assigned = False
|
| 200 |
+
|
| 201 |
+
# 1. Categorize by URL segments/hints
|
| 202 |
for c, hints in CATEGORY_HINTS.items():
|
| 203 |
if any(h in norm for h in hints):
|
| 204 |
+
# Format the last part of the URL as a Title
|
| 205 |
+
slug = url.rstrip('/').split('/')[-1]
|
| 206 |
+
title = format_url_to_title(slug)
|
| 207 |
+
if title:
|
| 208 |
+
cats[c].append(title)
|
| 209 |
assigned = True
|
| 210 |
break
|
| 211 |
+
|
| 212 |
+
# 2. Fallback Categorization
|
| 213 |
if not assigned:
|
| 214 |
parts = list(filter(None, urlparse(url).path.split('/')))
|
| 215 |
if parts:
|
|
|
|
| 216 |
cat_candidate = parts[0].title()
|
| 217 |
+
slug = parts[-1]
|
| 218 |
+
title = format_url_to_title(slug)
|
| 219 |
+
|
| 220 |
if len(cat_candidate) < 20 and not any(x in cat_candidate.lower() for x in ['202', '0', '1']):
|
| 221 |
+
cats[cat_candidate].append(title)
|
| 222 |
else:
|
| 223 |
+
cats["General"].append(title)
|
| 224 |
+
|
| 225 |
+
# Remove duplicates and sort
|
| 226 |
+
for k in cats:
|
| 227 |
+
cats[k] = sorted(list(set(cats[k])))
|
| 228 |
+
|
| 229 |
return dict(cats)
|
| 230 |
|
| 231 |
# ============================================================================
|
|
|
|
| 243 |
|
| 244 |
selected_formats = random.sample(formats[stage], 3)
|
| 245 |
|
|
|
|
| 246 |
style_prefix = ""
|
| 247 |
if "GPT" in model_name: style_prefix = "Data-Driven"
|
| 248 |
elif "Claude" in model_name: style_prefix = "Nuanced & Strategic"
|
|
|
|
| 262 |
return "\n".join(ideas)
|
| 263 |
|
| 264 |
def score_model_output(output, topic, stage):
|
|
|
|
|
|
|
| 265 |
score = random.randint(85, 98)
|
|
|
|
|
|
|
| 266 |
metrics = {
|
| 267 |
"Topic Relevance": random.randint(18, 20),
|
| 268 |
"Stage Alignment": random.randint(18, 20),
|
|
|
|
| 280 |
|
| 281 |
results = []
|
| 282 |
|
|
|
|
| 283 |
for model in AI_MODELS:
|
| 284 |
response = generate_model_response(model, topic, stage)
|
| 285 |
score, metrics = score_model_output(response, topic, stage)
|
|
|
|
| 290 |
"response": response
|
| 291 |
})
|
| 292 |
|
|
|
|
| 293 |
results.sort(key=lambda x: x['score'], reverse=True)
|
| 294 |
|
|
|
|
| 295 |
comparison_html = """
|
| 296 |
<table style="width:100%; border-collapse: collapse; margin-bottom: 20px;">
|
| 297 |
<tr style="background: #f3f4f6; border-bottom: 2px solid #e5e7eb;">
|
|
|
|
| 317 |
"""
|
| 318 |
comparison_html += "</table>"
|
| 319 |
|
|
|
|
| 320 |
details_html = ""
|
| 321 |
for res in results:
|
| 322 |
details_html += f"""
|
|
|
|
| 345 |
# ============================================================================
|
| 346 |
|
| 347 |
def generate_pill_html(items, color_class, empty_msg):
|
|
|
|
| 348 |
if not items:
|
| 349 |
return f'<p style="color: #666; font-style: italic;">{empty_msg}</p>'
|
| 350 |
|
|
|
|
|
|
|
| 351 |
colors = {
|
| 352 |
"orange": {"bg": "#fff7ed", "border": "#fdba74", "text": "#c2410c"},
|
| 353 |
"green": {"bg": "#f0fdf4", "border": "#86efac", "text": "#15803d"}
|
| 354 |
}
|
| 355 |
c = colors[color_class]
|
| 356 |
|
| 357 |
+
html = '<div style="display: flex; flex-wrap: wrap; gap: 10px;">'
|
| 358 |
for item in items:
|
|
|
|
| 359 |
clean_text = item.split('(')[0].strip() if '(' in item else item
|
| 360 |
link_url = f"https://www.google.com/search?q={clean_text.replace(' ', '+')}"
|
| 361 |
|
| 362 |
html += f"""
|
| 363 |
<a href="{link_url}" target="_blank" style="text-decoration: none;">
|
| 364 |
+
<span style="background-color: {c['bg']}; border: 1px solid {c['border']}; color: {c['text']}; padding: 6px 14px; border-radius: 20px; font-size: 0.9em; font-weight: 500;">
|
| 365 |
+
{clean_text} ↗
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
</span>
|
| 367 |
</a>
|
| 368 |
"""
|
|
|
|
| 376 |
t = set()
|
| 377 |
for u in urls:
|
| 378 |
slug = path_norm(u)
|
|
|
|
| 379 |
t.update([w for w in slug.split('-') if len(w)>3 and not w.isdigit()])
|
| 380 |
return t
|
| 381 |
|
|
|
|
| 384 |
for c in competitors:
|
| 385 |
all_comp_tokens.update(get_tokens(c['urls']))
|
| 386 |
|
|
|
|
| 387 |
gaps = sorted(list(all_comp_tokens - my_tokens))
|
|
|
|
| 388 |
strengths = sorted(list(my_tokens - all_comp_tokens))
|
| 389 |
|
|
|
|
| 390 |
if filter_topic:
|
| 391 |
gaps = [g for g in gaps if filter_topic.lower() in g]
|
| 392 |
strengths = [s for s in strengths if filter_topic.lower() in s]
|
| 393 |
else:
|
|
|
|
| 394 |
gaps = gaps[:25]
|
| 395 |
strengths = strengths[:25]
|
| 396 |
|
| 397 |
+
gap_html = generate_pill_html(gaps, "orange", "✅ No gaps found for this topic!")
|
| 398 |
+
str_html = generate_pill_html(strengths, "green", "⚪ No unique strengths found.")
|
| 399 |
|
| 400 |
return gap_html, str_html
|
| 401 |
|
| 402 |
def generate_strategic_summary(gaps, strengths):
|
|
|
|
| 403 |
return f"""
|
| 404 |
### 🧠 AI Strategic Recommendation
|
| 405 |
|
| 406 |
+
1. **Attack the Gaps:** Competitors are ranking for **{len(gaps)} topics** you missed.
|
| 407 |
+
*Priority:* Create "Hub & Spoke" content clusters around these keywords.
|
| 408 |
|
| 409 |
2. **Defend Strengths:** You have **{len(strengths)} unique authority areas**.
|
| 410 |
*Action:* Update these pages with fresh data to maintain your lead.
|
| 411 |
|
| 412 |
+
3. **Content Velocity:** Consider increasing publication frequency by **20%** to match competitor output.
|
| 413 |
"""
|
| 414 |
|
| 415 |
# ============================================================================
|
|
|
|
| 423 |
cats = categorize_links(urls)
|
| 424 |
state['urls'] = urls
|
| 425 |
cat_list = ["All"] + sorted(list(cats.keys()))
|
| 426 |
+
return urls, cats, msg, gr.update(choices=cat_list, value="All"), gr.update(choices=[])
|
| 427 |
|
| 428 |
def ui_update_topics(category, categories_data):
|
| 429 |
if not categories_data: return gr.update(choices=[])
|
|
|
|
| 436 |
if not urls: return "Fetch data first.", None, None
|
| 437 |
if not topic: return "Select a topic.", None, None
|
| 438 |
|
| 439 |
+
# Match logic (using fuzzy match on normalized slug vs normalized title)
|
| 440 |
+
topic_slug = normalize_str(topic)
|
| 441 |
+
matched = [u for u in urls if topic_slug in path_norm(u)]
|
| 442 |
|
| 443 |
stages = [classify_stage(u) for u in matched]
|
| 444 |
tot = len(stages) or 1
|
|
|
|
| 451 |
urls, msg = fetch_sitemap_links(url)
|
| 452 |
domain = urlparse(url if url.startswith('http') else f'https://{url}').netloc
|
| 453 |
|
|
|
|
| 454 |
stages = [classify_stage(u) for u in urls]
|
| 455 |
tot = len(stages) or 1
|
| 456 |
stats = {'TOFU': (stages.count('TOFU')/tot)*100, 'MOFU': (stages.count('MOFU')/tot)*100, 'BOFU': (stages.count('BOFU')/tot)*100, 'Count': len(urls)}
|
|
|
|
| 461 |
txt = "\n".join([f"✅ {c['domain']}: {c['stats']['Count']} pages" for c in current_comps])
|
| 462 |
return current_comps, txt, ""
|
| 463 |
|
| 464 |
+
def ui_comp_search(user_urls, competitors, gap_topic, str_topic):
|
| 465 |
+
if not user_urls: return "Fetch user data first.", ""
|
| 466 |
+
|
| 467 |
+
g_html, _ = analyze_gaps_strengths(user_urls, competitors, gap_topic)
|
| 468 |
+
_, s_html = analyze_gaps_strengths(user_urls, competitors, str_topic)
|
| 469 |
+
|
| 470 |
+
return g_html, s_html
|
| 471 |
+
|
| 472 |
def ui_comp_master(user_urls, competitors):
|
| 473 |
if not user_urls: return None, "", "", ""
|
| 474 |
|
|
|
|
| 516 |
|
| 517 |
with gr.Row():
|
| 518 |
cat_dd = gr.Dropdown(label="Category", choices=[], allow_custom_value=True)
|
| 519 |
+
topic_dd = gr.Dropdown(label="Topic (Formatted Title)", choices=[], allow_custom_value=True)
|
| 520 |
analyze_btn = gr.Button("Analyze Topic Health")
|
| 521 |
|
| 522 |
with gr.Row():
|
|
|
|
| 583 |
# 3. Competitors
|
| 584 |
add_comp.click(ui_add_comp, [comp_url, state_comps], [state_comps, comp_list, comp_url])
|
| 585 |
|
| 586 |
+
# Run Analysis
|
| 587 |
run_comp_btn.click(ui_comp_master, [state_urls, state_comps], [radar, gap_html, str_html, strat_rec])
|
| 588 |
|
| 589 |
# Search Gaps/Strengths
|