Blame for response_modes.py

import time
from langchain_community.tools import DuckDuckGoSearchRun
from cerebras.cloud.sdk import Cerebras
from langchain_openai import ChatOpenAI
import os
import requests
import asyncio
import aiohttp
import textwrap
import re  
from datetime import datetime, timedelta
from auth import save_chats
from bs4 import BeautifulSoup
from openai import OpenAI
from duckduckgo_search import DDGS
from PyPDF2 import PdfReader
from docx import Document

from urllib.parse import urlparse, quote
from retry import retry  # For retrying flaky operations
from groq import Groq  # For the second AI (Qwen-3-32B)

# ================== PHILIPPINE LOCAL + MAX SEARCH + DEEP-THINK LENIENT FILTER ==================
PHILIPPINE_LOCATIONS = [
    "bacolod", "negros occidental", "negros", "iloilo", "cebu", "davao", "manila", "quezon",
    "makati", "pasig", "taguig", "cagayan de oro", "zamboanga", "general santos", "dumaguete",
    "tacloban", "ormoc", "dipolog", "pagadian", "roxas city", "kalibo", "antique", "capiz",
    "guimaras", "siquijor", "bohol", "leyte", "samar", "masbate"
]

def is_philippine_local_query(query: str) -> bool:
    q = query.lower()
    return any(loc in q for loc in PHILIPPINE_LOCATIONS)

def enhance_query(query: str) -> str:
    today_date = datetime.now().strftime("%B %d, %Y")
    q = query.lower().strip()
    if q in ["news today", "news", "today", "latest news", "headlines", "breaking news"]:
        return f"top world news headlines {today_date} breaking news"
    if is_philippine_local_query(query):
        return f"{query} Philippines latest news {today_date}"
    if any(w in q for w in ['today', 'latest', 'breaking', 'current', 'now']):
        return f"{query} latest {today_date}"
    return query

def get_cutoff_days(query: str) -> int:
    """Smart cutoff: very strict for "today", lenient for deep sub-queries"""
    q = query.lower()
    if any(x in q for x in ["today", "breaking", "just now", "live", "happening now"]):
        return 3
    if any(x in q for x in ["this week", "latest", "recent update", "current"]):
        return 10
    if any(x in q for x in ["controversy", "alternative", "debate", "criticism", "scandal", "views"]):
        return 90   # Deep Think needs these
    if any(x in q for x in ["solution", "how to", "fix", "analysis", "details", "updates", "mainstream"]):
        return 45
    if is_philippine_local_query(query) or "news" in q:
        return 12
    return 35

def get_domain(url: str) -> str:
    try:
        return urlparse(url).netloc.lower().replace("www.", "").replace("m.", "")
    except:
        return ""

def domain_score(url: str, is_ph: bool) -> int:
    dom = get_domain(url)
    RELIABLE_PH = {"inquirer.net", "rappler.com", "philstar.com", "gmanetwork.com", "abs-cbnnews.com",
                   "manilatimes.net", "cnnphilippines.com", "mb.com.ph", "businessworld.com.ph", "pna.gov.ph"}
    RELIABLE_GLOBAL = {"reuters.com", "apnews.com", "bbc.com", "nytimes.com", "bloomberg.com", "theguardian.com", "aljazeera.com"}
    if dom in (RELIABLE_PH if is_ph else RELIABLE_GLOBAL):
        return 95
    if dom.endswith(".ph") or "gov.ph" in dom:
        return 75
    if any(g in dom for g in RELIABLE_GLOBAL):
        return 85
    return 45

def parse_ddgs_date(date_str: str):
    if not date_str or date_str.lower() in ["recent", ""]:
        return datetime.now()
    date_str = date_str.lower()
    now = datetime.now()
    match = re.search(r"(\d+)\s*(hour|day|week|month|min)", date_str)
    if match:
        num = int(match.group(1))
        unit = match.group(2)
        if "min" in unit or "hour" in unit: return now - timedelta(hours=max(1, num))
        if "day" in unit: return now - timedelta(days=num)
        if "week" in unit: return now - timedelta(weeks=num)
        if "month" in unit: return now - timedelta(days=num * 30)
    try:
        return datetime.strptime(date_str[:10], "%Y-%m-%d")
    except:
        return now

async def fetch_full_content_async(session, url: str):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
        async with session.get(url, headers=headers, timeout=12) as response:
            if response.status != 200:
                return None
            text = await response.text()
            soup = BeautifulSoup(text, 'html.parser')
            for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
                tag.decompose()
            content = ' '.join(p.get_text(strip=True) for p in soup.find_all('p') if len(p.get_text(strip=True)) > 25)
            return content[:4800] if len(content) > 100 else None
    except Exception as e:
        return None

@retry(tries=2, delay=1)  # Retry on failures
async def web_search_async(query, max_results=120):
    """MAX websites + 3-stage fallback = Grok-level deep search that NEVER returns empty"""
    try:
        enhanced = enhance_query(query)
        is_ph = is_philippine_local_query(query)
        cutoff_days = get_cutoff_days(query)

        # === DYNAMIC TODAY DATE (used in all fallbacks) ===
        today_str = datetime.now().strftime("%B %d, %Y")

        with DDGS() as ddgs:
            raw = []
            try: raw.extend(list(ddgs.news(keywords=enhanced, max_results=60, safesearch="off", region="ph-en" if is_ph else "wt-wt")))
            except: pass
            try: raw.extend(list(ddgs.text(keywords=enhanced, max_results=80)))
            except: pass

        # Dedup + rank
        seen = set()
        unique = [r for r in raw if (url := r.get('url') or r.get('href','')) and url not in seen and url.startswith('http') and (seen.add(url) or True)]
        unique.sort(key=lambda r: (domain_score(r.get('url') or '', is_ph), parse_ddgs_date(r.get('date','')).timestamp()), reverse=True)

        # Async fetch full content (limit to top 20 to avoid overload)
        async with aiohttp.ClientSession() as session:
            tasks = [fetch_full_content_async(session, r.get('url') or r.get('href', '')) for r in unique[:20]]
            full_contents = await asyncio.gather(*tasks, return_exceptions=True)

        formatted = []
        for idx, (r, full) in enumerate(zip(unique[:100], full_contents + [None] * (100 - len(full_contents)))):
            if isinstance(full, Exception): full = None
            url = r.get('url') or r.get('href', '')
            pub_date = parse_ddgs_date(r.get('date', ''))
            age_days = (datetime.now() - pub_date).days
            if age_days > cutoff_days + 15: continue

            if full and len(full) > 120:
                title = r.get('title', 'No title')
                snippet = r.get('body', '')[:800]
                age_str = "today" if age_days <= 1 else f"{age_days}d ago"
                entry = f"[{len(formatted)+1}] **{title}** ({age_str})\nSnippet: {snippet[:380]}...\n[URL: {url}]"
                formatted.append(entry)
                if len(formatted) >= 18: break

        # STAGE 2 & 3 FALLBACK — guarantees results
        if len(formatted) < 10:
            relaxed = cutoff_days * 3
            for r in unique[30:160]:
                if any(r.get('url','') in e for e in formatted): continue
                pub_date = parse_ddgs_date(r.get('date', ''))
                if (datetime.now() - pub_date).days > relaxed: continue
                full = await fetch_full_content_async(session, r.get('url') or r.get('href',''))  # Single async fetch for fallback
                if full:
                    title = r.get('title', 'No title')
                    snippet = r.get('body', '')[:800]
                    entry = f"[{len(formatted)+1}] **{title}** ({(datetime.now()-pub_date).days}d ago - relaxed)\nSnippet: {snippet[:380]}...\n[URL: {r.get('url') or r.get('href','')}]"
                    formatted.append(entry)
                    if len(formatted) >= 16: break

        # === FIXED: Dynamic date fallback ===
        return formatted or [f"Latest available data for \"{query}\" as of {today_str} (broad search used)."]

    except Exception as e:
        today_str = datetime.now().strftime("%B %d, %Y")  # also dynamic in error case
        return [f"Search error: {str(e)[:120]}... Using latest available data as of {today_str}."]
        
def web_search(query, max_results=120):
    """Sync wrapper for async web_search"""
    return asyncio.run(web_search_async(query, max_results))

def extract_file_text(file_paths):
    if not file_paths:
        return ""
    texts = []
    for path in file_paths:
        try:
            if path.lower().endswith('.txt'):
                with open(path, 'r', encoding='utf-8') as f:
                    texts.append(f"--- {os.path.basename(path)} ---\n{f.read()}")
            elif path.lower().endswith('.pdf'):
                reader = PdfReader(path)
                text = "\n".join(page.extract_text() or "" for page in reader.pages)
                texts.append(f"--- {os.path.basename(path)} ---\n{text}")
            elif path.lower().endswith('.docx'):
                doc = Document(path)
                text = "\n".join(para.text for para in doc.paragraphs)
                texts.append(f"--- {os.path.basename(path)} ---\n{text}")
        except Exception as e:
            texts.append(f"--- {os.path.basename(path)} ---\n[Error reading file: {str(e)}]")
    return "\n\n".join(texts)[:12000]

def generate_queries(client, current_model, message, today_date):
    """Helper to generate smart queries for Deep Think"""
    try:
        plan_prompt = f"""Generate 3-6 complete, concise and correct keyword phrases (3-8 words max, NO operators, NO site except if asked, NO OR/AND) for deep live search on: "{quote(message)}".
Include time words (today, 2026, latest, March 2026, etc.) where helpful.
If a quiz is asked then be precise and straight to the point when answering.
Make them varied: mainstream news, global news, background, controversy, updates, global/local, different angles.
One phrase per line only. No numbers, no explanations except if asked."""
        plan = client.chat.completions.create(
            model=current_model,
            messages=[{"role": "user", "content": plan_prompt}],
            max_tokens=250, temperature=0.4
        )
        return [line.strip() for line in plan.choices[0].message.content.splitlines() if line.strip()][:6]
    except Exception:
        base = message if len(message) < 60 else message[:55]
        return [
            f"{base} latest news",
            f"{base} March 2026",
            f"{base} controversy",
            f"{base} updates today",
            f"{base} analysis",
            f"{base} mainstream",
            f"{base} world",
            f"{base} usa",
            f"{base} global"
        ]

# Setup for the second AI model (Qwen-3-32B)
client_qwen = Groq()  # For using the Qwen-3-32B model

def clean_response(text: str) -> str:
    """Strip leaked <think> blocks and extra newlines"""
    if not text:
        return ""
    text = re.sub(r'<think>.*?</think>', '', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'```(?:think|reasoning).*?```', '', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()


def format_sources(sources_list, max_sources=7):
    """Super clean, beautiful sources for both Philippine & global news"""
    if not sources_list:
        return ""
    
    formatted = []
    for i, entry in enumerate(sources_list[:max_sources], 1):
        title = "Source"
        snippet = ""
        url = "#"
        
        if '**' in entry:
            title_match = re.search(r'\*\*(.+?)\*\*', entry)
            if title_match:
                title = title_match.group(1).strip()
        
        if 'Snippet:' in entry:
            snippet_part = entry.split('Snippet:')[1].split('\n')[0]
            snippet = snippet_part.strip()[:260] + "..." if len(snippet_part) > 260 else snippet_part.strip()
        
        url_match = re.search(r'\[URL:\s*(https?://[^\]]+)', entry)
        if url_match:
            url = url_match.group(1).strip()
        
        formatted.append(f"{i}. **{title}**  \n   {snippet}\n   [Read full →]({url})")
    
    return "\n\n".join(formatted)


def qwen_reasoning(message: str, search_context: str, chat_context: str = "") -> str:
    """Ultra-strict: 100% obeys chat instructions + only real data + specific"""
    system_instruction = ""
    if chat_context and str(chat_context).strip():
        system_instruction = f"""[PERSISTENT CHAT INSTRUCTION — MUST OBEY 100% FOR THIS CHAT]
{chat_context}

→ The ENTIRE answer must follow this instruction exactly (e.g. use only Filipino, be funny, be concise, etc.).
→ If it says "Use Filipino", reply 100% in Filipino.
→ If it says "be funny", add jokes in every response.
"""

    prompt = textwrap.dedent(f"""\
{system_instruction}You are a precise news analyst. Use ONLY the real-time search data provided below. Do not add anything from your knowledge.

Real-time search data:
{search_context}

User asked: "{message}"

STRICT RULES:
- Be extremely specific: name exact events, people, places, numbers, dates from the data only.
- NEVER generalize, never say "details emerging", "potential", "focus areas", "stay tuned", or make up 2026 events.
- If search data has no specific news, say exactly: "Walang major specific events sa latest searches para ngayon. Ito ang top general headlines mula sa data:"
- Prioritize Philippine sources when query mentions Philippines.
- Use clear markdown with bullets and bold facts.
- Never output <think> or reasoning.

Final answer:""")

    try:
        completion = client_qwen.chat.completions.create(
            model="qwen2-72b-instruct",
            messages=[{"role": "user", "content": prompt}],
            temperature=0.45,   # even more factual
            max_tokens=2000,
            top_p=0.88,
            stream=False,
        )
        return clean_response(completion.choices[0].message.content or "")
    except Exception:
        return "Limited verified updates available right now."

# Cerebras Reviewer (strong reasoning model)
cerebras_client = Cerebras(
    api_key=os.environ.get("CEREBRAS_API_KEY")
)

# ================== MAIN CHAT STREAM (fixed and refined for all modes) ==================
def chat_stream(message, history, current_model, logged_in_username, user_chats, current_chat_title, mode, auth_globals, file_paths=None, context=""):
    if file_paths and isinstance(file_paths, list) and len(file_paths) > 5:
        history.append({"role": "assistant", "content": "❌ Maximum 5 files allowed per message. Please remove some and try again."})
        yield history
        return

    client = OpenAI(base_url="https://integrate.api.nvidia.com/v1", api_key=os.environ.get("API_KEY"))
    
    file_text = extract_file_text(file_paths) if file_paths else ""
    
    history = history or []
    if not logged_in_username or not current_model or not current_chat_title:
        history.append({"role": "assistant", "content": "⚠️ Please login, load model, and select/create a chat first!"})
        yield history
        return

    user_input = quote(message)
    if file_text:
        user_input += f"\n\n--- Attached files (up to 5) ---\n{file_text}"

    history.append({"role": "user", "content": user_input})
    yield history

    history.append({"role": "assistant", "content": ""})
    yield history

    today_date = datetime.now().strftime("%B %d, %Y")

    # === STRONG CONTEXT HANDLING (works in EVERY mode) ===
    context_instruction = ""
    if context and str(context).strip():
        ctx = str(context).strip()
        context_instruction = f"""[PERSISTENT CHAT INSTRUCTION — ALWAYS FOLLOW FOR THIS CHAT ONLY]
{ctx}

You MUST follow the above instruction naturally in EVERY response."""

    system_context = [{"role": "system", "content": context_instruction}] if context_instruction else []

    if mode == "💻 Coding Mode":
        # Gather persistent files
        coding_files_str = ""
        current_chat = next((chat for chat in (user_chats or []) if chat.get("title") == current_chat_title), None)
        if current_chat and current_chat.get("coding_files"):
            coding_files_str = "\n\n=== PERSISTENT PROJECT FILES ===\n"
            for fname, data in current_chat["coding_files"].items():
                coding_files_str += f"\n--- {fname} ---\n{data['content']}\n"

        full_task = f"User request: {user_input}\n\nProject files:{coding_files_str}"

        # === STEP 1: RESEARCHER ===
        history[-1]["content"] = "🔍 Researcher is searching StackOverflow, Reddit, GitHub..."
        yield history

        research_results = web_search(f"best solution {full_task} site:stackoverflow.com OR site:reddit.com/r/learnpython OR site:github.com", max_results=15)

        # === STEP 2: CODER ===
        history[-1]["content"] += "\n🤖 Coder is writing the code..."
        yield history

        coder_prompt = f"""You are an expert senior software engineer.

Project files:
{coding_files_str}

Research findings:
{"\n\n".join(research_results[:8]) if research_results else "No external research available."}

User request: {user_input}

Write clean, well-commented, production-ready code."""

        coder_response = client.chat.completions.create(
            model=current_model,
            messages=system_context + [{"role": "user", "content": coder_prompt}],
            temperature=0.3,
            max_tokens=4096,
            stream=False
        ).choices[0].message.content

        history[-1]["content"] += f"\n\n🤖 **Coder**:\n\n{coder_response}"
        yield history

        # === STEP 3: REVIEWER ===
        history[-1]["content"] += "\n🧠 Cerebras Reviewer is doing deep analysis..."
        yield history

        reviewer_prompt = f"""You are an extremely strict senior code reviewer.

Code to review:
{coder_response[:3000]}

User request:
{user_input[:500]}

Review strictly:
- Bugs / logic errors
- Security risks
- Performance issues
- Bad practices
- Suggest fixes
Then give the FINAL polished code.

If good, say "✅ Looks solid" and return it unchanged."""

        try:
            cerebras_response = cerebras_client.chat.completions.create(
                messages=system_context + [{"role": "user", "content": reviewer_prompt}],
                model="llama3.1-70b",
                stream=False,
                temperature=0.3,
                max_tokens=8192
            ).choices[0].message.content or "✅ Reviewer found no major issues."
        except Exception as e:
            cerebras_response = f"Reviewer failed: {str(e)[:200]}\n\nOriginal code kept."

        history[-1]["content"] += f"\n\n🧠 **Cerebras Reviewer**:\n\n{cerebras_response}"
        yield history

        # Final output
        final_text = f"\n\n✅ **Final Polished Code**\n\n{cerebras_response}\n\n**Original version:**\n{coder_response}"
        history[-1]["content"] += final_text
        yield history

    elif mode == "Deep Think":
        history[-1]["content"] = "🧠 Deep Think mode active — searching and reasoning..."
        yield history

        # AI1 (NVIDIA): Generate queries
        queries = generate_queries(client, current_model, message, today_date)

        # Perform searches in parallel
        search_results = []
        for q in queries:
            search_results.extend(web_search(q, max_results=20))

        search_context = "\n\n".join(search_results[:15])  # Limit to top results

        history[-1]["content"] += f"\n\n🔍 Searches done on: {', '.join(queries)}"
        yield history

        # AI2 (Qwen via Groq): Initial reasoning
        initial_reasoning = qwen_reasoning(message, search_context, context)

        history[-1]["content"] += f"\n\n🤔 Initial Analysis:\n{initial_reasoning}"
        yield history

        # AI1 (Cerebras): Review and refine the reasoning
        review_prompt = f"""Review this initial analysis strictly for accuracy, completeness, and adherence to data:

Initial Analysis:
{initial_reasoning}

Search Data:
{search_context[:2000]}

User Query: {message}

Refine it: Add missing details, correct errors, ensure no hallucinations. Provide final polished response."""

        try:
            refined_response = cerebras_client.chat.completions.create(
                messages=system_context + [{"role": "user", "content": review_prompt}],
                model="llama3.1-70b",
                stream=False,
                temperature=0.4,
                max_tokens=2000
            ).choices[0].message.content
        except Exception as e:
            refined_response = f"Refinement failed: {str(e)[:200]}\n\nUsing initial reasoning."

        history[-1]["content"] += f"\n\n🧠 Refined Final Response:\n{refined_response}\n\nSources:\n{format_sources(search_results)}"
        yield history

    elif mode == "Fast Search":
        history[-1]["content"] = "⚡ Fast Search mode — quick web lookup..."
        yield history

        # Single AI (NVIDIA): Enhance query and search
        enhanced_query = enhance_query(message)
        search_results = web_search(enhanced_query, max_results=10)
        search_context = "\n\n".join(search_results)

        # Generate response using one AI
        prompt = f"""Summarize key facts from this search data for: "{message}"

Data:
{search_context}

Be concise, use bullets."""

        response = client.chat.completions.create(
            model=current_model,
            messages=system_context + [{"role": "user", "content": prompt}],
            temperature=0.5,
            max_tokens=1000,
            stream=False
        ).choices[0].message.content

        history[-1]["content"] += f"\n\n{response}\n\nSources:\n{format_sources(search_results, max_sources=5)}"
        yield history

    else:  # Default Chat
        # Single AI (NVIDIA): Normal conversation
        full_prompt = "\n".join([msg["content"] for msg in history[-5:]])  # Last 5 messages for context

        response = client.chat.completions.create(
            model=current_model,
            messages=system_context + [{"role": "user", "content": full_prompt}],
            temperature=0.7,
            max_tokens=1500,
            stream=False
        ).choices[0].message.content

        history[-1]["content"] = response
        yield history

    # Auto-save (common for all modes)
    if logged_in_username and user_chats:
        for chat in user_chats:
            if chat["title"] == current_chat_title:
                chat["history"] = history
                break
        save_chats(logged_in_username, user_chats, auth_globals)