Update app.py

app.py

@@ -3,6 +3,7 @@ import json
 import re
 import uuid
 import threading
+from concurrent.futures import ThreadPoolExecutor
 import requests
 import feedparser
 from bs4 import BeautifulSoup
@@ -13,6 +14,9 @@ from llama_cpp import Llama
 app = Flask(__name__)
 CORS(app)
 
+MAX_CONCURRENT_USERS = 20
+executor = ThreadPoolExecutor(max_workers=MAX_CONCURRENT_USERS)
+
 MODEL_REPO = "unsloth/Qwen3-0.6B-GGUF"
 MODEL_FILE = "Qwen3-0.6B-Q4_K_M.gguf"
 N_CTX = 2048
@@ -68,6 +72,7 @@ def crawl_site(url):
     try:
         headers = {'User-Agent': 'Mozilla/5.0 (compatible; JuskeoBot/1.0)'}
         resp = requests.get(url, timeout=15, headers=headers, allow_redirects=True)
+        resp.raise_for_status()
         soup = BeautifulSoup(resp.text, 'html.parser')
 
         title = soup.find('title')
@@ -276,13 +281,13 @@ def run_crawl_task(task_id, url, username, webhook):
 
         if "error" in crawl_data:
             tasks[task_id]['status'] = 'error'
-            tasks[task_id]['error'] = crawl_data['error']
+            tasks[task_id]['error'] = "Crawl failed: " + crawl_data['error']
             return
 
-        text_content = crawl_data.get("text_preview", "")
+        text_content = crawl_data.get("body_text", "") or crawl_data.get("text_preview", "")
 
         tasks[task_id]['progress'] = 50
-        analysis_prompt = f"Analyze this website:\n\nTitle: {crawl_data['title']}\nMeta: {crawl_data['meta_description']}\nContent: {text_content[:2000]}\nIssues: {json.dumps(crawl_data['seo_issues'])}"
+        analysis_prompt = f"Analyze this website:\n\nTitle: {crawl_data['title']}\nMeta: {crawl_data['meta_description']}\nH1: {', '.join(crawl_data['h1'][:5])}\nH2: {', '.join(crawl_data['h2'][:10])}\nContent: {text_content[:2000]}\nIssues: {json.dumps(crawl_data['seo_issues'])}"
         analysis_response = run_llm(analysis_prompt, system_prompt=SEO_PROMPT, max_new_tokens=1024)
         analysis = parse_json_from_llm(analysis_response)
         tasks[task_id]['progress'] = 70
@@ -309,12 +314,39 @@ def run_crawl_task(task_id, url, username, webhook):
 
         llm_mentions = {'chatgpt': 0, 'perplexity': 0, 'gemini': 0, 'claude': 0}
 
-        tasks[task_id]['progress'] = 85
+        tasks[task_id]['progress'] = 80
 
         blog_posts = []
         blog_topic = crawl_data['title'] or 'Technology Trends'
-        blog_prompt = f"Write a comprehensive blog post about: {blog_topic}\n\nKeywords: {', '.join(keywords[:5])}\nSite Context: {crawl_data['title']}\nWrite 1500+ words in markdown."
+
+        news_items = fetch_news_with_content(blog_topic, max_results=3)
+        news_text = ""
+        if news_items:
+            news_text = "\n\n=== REFERENCE ARTICLES ===\n"
+            for i, n in enumerate(news_items[:3], 1):
+                news_text += f"\n--- Article {i}: {n.get('title', '')} ({n.get('source', '')}) ---\n"
+                news_text += f"{n.get('content', '')[:2000]}\n"
+
+        blog_prompt = f"""Write a comprehensive blog post about: {blog_topic}
+
+Keywords: {', '.join(keywords[:5])}
+Site Context: {crawl_data['title']}
+Site Description: {crawl_data['meta_description']}
+Headings: {', '.join(crawl_data['h2'][:5])}
+{news_text}
+
+RULES:
+- Write 2000-3000 words MINIMUM. Long-form authoritative article.
+- Use ## for main sections (8-12 sections), ### for subsections.
+- Each section 200-300 words of REAL content.
+- Include data, statistics, examples from reference articles.
+- Include FAQ section at the end with 5+ questions.
+- Naturally place target keywords throughout.
+- NO thinking tags. Output ONLY the markdown blog post."""
+
+        tasks[task_id]['progress'] = 85
         blog_response = run_llm(blog_prompt, system_prompt=BLOG_PROMPT, max_new_tokens=4096)
+        blog_response = strip_thinking(blog_response)
         word_count = len(blog_response.split())
         title_match = re.search(r'^#\s+(.+)$', blog_response, re.MULTILINE)
         blog_title = title_match.group(1) if title_match else blog_topic
@@ -344,9 +376,10 @@ def run_crawl_task(task_id, url, username, webhook):
             'keywords': [{'keyword': k, 'volume': 0, 'position': i+1} for i, k in enumerate(keywords[:20])],
             'pages': pages_data,
             'blog_posts': blog_posts,
+            'news_used': news_items,
             'schema_types': schema_types,
             'llm_mentions': llm_mentions,
-            'crawled_at': __import__('datetime').datetime.utcnow().isoformat() + 'Z',
+            'crawled_at': __import__('datetime').datetime.now(__import__('datetime').timezone.utc).isoformat(),
             'llm_analysis': analysis,
         }
 
@@ -355,10 +388,15 @@ def run_crawl_task(task_id, url, username, webhook):
         tasks[task_id]['result'] = result
 
         if webhook:
-            try:
-                requests.post(webhook, json={'username': username, 'result': result}, timeout=30)
-            except:
-                pass
+            for attempt in range(3):
+                try:
+                    resp = requests.post(webhook, json={'username': username, 'result': result}, timeout=30)
+                    if resp.status_code < 400:
+                        break
+                except:
+                    if attempt < 2:
+                        import time
+                        time.sleep(2 * (attempt + 1))
 
     except Exception as e:
         tasks[task_id]['status'] = 'error'
@@ -383,9 +421,7 @@ def index():
             'error': None,
         }
 
-        t = threading.Thread(target=run_crawl_task, args=(task_id, url, username, webhook))
-        t.daemon = True
-        t.start()
+        t = executor.submit(run_crawl_task, task_id, url, username, webhook)
 
         return f'''<html><head><meta http-equiv="refresh" content="0;url=/status?task_id={task_id}"></head><body>
         <script>let t="{task_id}";window.location.href="/status?task_id="+t;</script>
@@ -401,11 +437,14 @@ def task_status():
         return jsonify({"status": "error", "message": "Unknown task"}), 404
 
     task = tasks[task_id]
-    return jsonify({
+    resp = {
         "status": task['status'],
         "progress": task['progress'],
         "pages_crawled": task.get('result', {}).get('pages_crawled', 0) if task['result'] else 0,
-    })
+    }
+    if task['status'] == 'error':
+        resp['error'] = task.get('error', 'Unknown error')
+    return jsonify(resp)
 
 
 @app.route('/result', methods=['GET'])
@@ -502,6 +541,9 @@ def generate_blog():
         if not topic:
             return jsonify({"error": "Topic is required"}), 400
 
+        if not news:
+            news = fetch_news_with_content(topic, max_results=5)
+
         news_text = ""
         if news:
             news_with_content = []
@@ -720,4 +762,4 @@ except Exception as e:
 
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=7860, debug=False)
+    app.run(host='0.0.0.0', port=7860, debug=False, threaded=True)