Update pages/facebook_extractor.py

pages/facebook_extractor.py

@@ -1,34 +1,490 @@
 # pages/facebook_extractor.py
 import streamlit as st
+import requests
+from bs4 import BeautifulSoup
+import json
+import re
+from datetime import datetime
+from typing import List, Dict
+import os
 
-def main():
-    st.set_page_config(page_title="Facebook Extractor", page_icon="📘", layout="wide")
+# Import your existing AI components
+from langchain_text_splitters import CharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+from langchain.schema import Document
+from langchain_community.llms import HuggingFaceHub
+
+st.set_page_config(
+    page_title="Facebook Data Extractor",
+    page_icon="📘",
+    layout="wide"
+)
+
+class FacebookPublicExtractor:
+    """Facebook public data extractor that works on Hugging Face"""
+    
+    def __init__(self):
+        self.session = requests.Session()
+        self.setup_session()
+    
+    def setup_session(self):
+        """Setup requests session with headers"""
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+        })
+    
+    def extract_public_data(self, url: str, data_type: str) -> Dict:
+        """Extract public data from Facebook URLs"""
+        try:
+            st.info(f"🌐 Accessing: {url}")
+            
+            # Validate URL type
+            url_type = self.analyze_facebook_url(url)
+            
+            response = self.session.get(url, timeout=15)
+            
+            if response.status_code != 200:
+                return {
+                    "error": f"Failed to access page (Status: {response.status_code})",
+                    "url_type": url_type,
+                    "status": "error"
+                }
+            
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            # Remove scripts and styles
+            for script in soup(["script", "style", "meta", "link"]):
+                script.decompose()
+            
+            # Extract basic information
+            text = soup.get_text()
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            clean_text = ' '.join(chunk for chunk in chunks if chunk)
+            
+            # Extract structured data
+            page_info = self._extract_page_info(soup, url, response)
+            content_blocks = self._extract_content_blocks(clean_text, url_type)
+            
+            return {
+                "page_info": page_info,
+                "content_blocks": content_blocks,
+                "url_type": url_type,
+                "extraction_time": datetime.now().isoformat(),
+                "data_type": data_type,
+                "status": "success"
+            }
+            
+        except Exception as e:
+            return {"error": f"Extraction failed: {str(e)}", "status": "error"}
+    
+    def analyze_facebook_url(self, url: str) -> str:
+        """Analyze Facebook URL type"""
+        url_lower = url.lower()
+        
+        if 'facebook.com/groups/' in url_lower:
+            return "Facebook Group (Limited public data)"
+        elif 'facebook.com/pages/' in url_lower or '/pages/' in url_lower:
+            return "Facebook Page (Public data available)"
+        elif 'facebook.com/events/' in url_lower:
+            return "Facebook Event (Limited access)"
+        elif 'facebook.com/marketplace/' in url_lower:
+            return "Facebook Marketplace"
+        elif 'facebook.com/' in url_lower and '/posts/' in url_lower:
+            return "Facebook Post"
+        else:
+            return "Facebook Profile/Page"
+    
+    def _extract_page_info(self, soup, url: str, response) -> Dict:
+        """Extract page information"""
+        title = soup.find('title')
+        meta_desc = soup.find('meta', attrs={'name': 'description'})
+        og_title = soup.find('meta', property='og:title')
+        og_description = soup.find('meta', property='og:description')
+        og_image = soup.find('meta', property='og:image')
+        
+        return {
+            "title": title.text.strip() if title else "Facebook Content",
+            "description": meta_desc['content'] if meta_desc else "",
+            "og_title": og_title['content'] if og_title else "",
+            "og_description": og_description['content'] if og_description else "",
+            "og_image": og_image['content'] if og_image else "",
+            "url": url,
+            "response_code": response.status_code,
+            "content_length": len(response.text),
+            "access_note": self._get_access_note(soup)
+        }
+    
+    def _extract_content_blocks(self, text: str, url_type: str) -> List[Dict]:
+        """Extract meaningful content blocks"""
+        blocks = []
+        
+        # Split into paragraphs/sentences
+        paragraphs = [p.strip() for p in text.split('.') if p.strip() and len(p.strip()) > 20]
+        
+        for i, paragraph in enumerate(paragraphs[:15]):  # Limit to first 15
+            content_type = self._analyze_content_type(paragraph, url_type)
+            
+            block = {
+                "id": i + 1,
+                "content": paragraph,
+                "length": len(paragraph),
+                "word_count": len(paragraph.split()),
+                "content_type": content_type,
+                "has_links": 'http' in paragraph.lower(),
+                "is_public_content": self._is_public_content(paragraph)
+            }
+            blocks.append(block)
+        
+        return blocks
+    
+    def _analyze_content_type(self, text: str, url_type: str) -> str:
+        """Analyze content type"""
+        text_lower = text.lower()
+        
+        if any(word in text_lower for word in ['event', 'date', 'time', 'location']):
+            return "event_info"
+        elif any(word in text_lower for word in ['group', 'community', 'member', 'join']):
+            return "community_info"
+        elif any(word in text_lower for word in ['marketplace', 'buy', 'sell', 'price']):
+            return "commerce"
+        elif any(word in text_lower for word in ['post', 'share', 'comment']):
+            return "social_content"
+        elif any(word in text_lower for word in ['login', 'sign in']):
+            return "authentication"
+        else:
+            return "general_content"
+    
+    def _is_public_content(self, text: str) -> bool:
+        """Check if content appears to be publicly accessible"""
+        text_lower = text.lower()
+        private_indicators = [
+            'log in to see', 'sign up to see', 'you must log in',
+            'private content', 'restricted access'
+        ]
+        
+        return not any(indicator in text_lower for indicator in private_indicators)
+    
+    def _get_access_note(self, soup) -> str:
+        """Get access level note"""
+        page_text = soup.get_text().lower()
+        
+        if any(phrase in page_text for phrase in ['log in', 'sign in', 'you must be logged in']):
+            return "Login required for full access"
+        elif 'content not available' in page_text:
+            return "Content not publicly available"
+        else:
+            return "Public content accessible"
+
+# AI Functions (same as your LinkedIn analyzer)
+def get_embeddings():
+    """Initialize embeddings"""
+    try:
+        embeddings = HuggingFaceEmbeddings(
+            model_name="sentence-transformers/all-MiniLM-L6-v2"
+        )
+        return embeddings
+    except Exception as e:
+        st.error(f"Embeddings error: {e}")
+        return None
+
+def get_llm():
+    """Initialize HuggingFace LLM"""
+    try:
+        api_key = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+        if not api_key:
+            st.error("HuggingFace API Key not found")
+            return None
+        
+        llm = HuggingFaceHub(
+            repo_id="mistralai/Mistral-7B-Instruct-v0.1",
+            huggingfacehub_api_token=api_key,
+            model_kwargs={
+                "temperature": 0.7,
+                "max_length": 512,
+                "max_new_tokens": 256,
+            }
+        )
+        return llm
+    except Exception as e:
+        st.error(f"LLM error: {e}")
+        return None
+
+def process_facebook_data(extracted_data):
+    """Process extracted data for AI analysis"""
+    if not extracted_data or extracted_data.get("status") != "success":
+        return None, []
+    
+    page_info = extracted_data['page_info']
+    content_blocks = extracted_data['content_blocks']
+    url_type = extracted_data['url_type']
+    
+    all_text = f"FACEBOOK DATA ANALYSIS\n{'='*50}\n\n"
+    all_text += f"📄 PAGE INFORMATION:\n"
+    all_text += f"Title: {page_info['title']}\n"
+    all_text += f"URL: {page_info['url']}\n"
+    all_text += f"URL Type: {url_type}\n"
+    all_text += f"Access: {page_info['access_note']}\n"
+    all_text += f"Extracted: {extracted_data['extraction_time']}\n\n"
+    
+    all_text += f"📊 CONTENT ANALYSIS:\n"
+    all_text += f"Content Blocks: {len(content_blocks)}\n"
+    all_text += f"Public Content: {sum(1 for b in content_blocks if b['is_public_content'])} blocks\n\n"
+    
+    for i, block in enumerate(content_blocks[:10]):
+        all_text += f"--- BLOCK {i+1} ---\n"
+        all_text += f"Type: {block['content_type']}\n"
+        all_text += f"Words: {block['word_count']} | Public: {block['is_public_content']}\n"
+        all_text += f"Content: {block['content']}\n\n"
+    
+    all_text += "="*50
     
-    st.title("📘 Facebook Data Extractor")
+    # Split into chunks
+    splitter = CharacterTextSplitter(
+        separator="\n",
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len
+    )
+    
+    chunks = splitter.split_text(all_text)
+    documents = [Document(page_content=chunk) for chunk in chunks]
+    
+    # Create vector store
+    try:
+        embeddings = get_embeddings()
+        if embeddings is None:
+            return None, []
+        vectorstore = FAISS.from_documents(documents, embeddings)
+        return vectorstore, chunks
+    except Exception as e:
+        st.error(f"Vector store failed: {e}")
+        return None, []
+
+def create_chatbot(vectorstore):
+    """Create conversational chatbot"""
+    try:
+        llm = get_llm()
+        if llm is None:
+            return None
+        
+        memory = ConversationBufferMemory(
+            memory_key="chat_history",
+            return_messages=True,
+            output_key="answer"
+        )
+        
+        chain = ConversationalRetrievalChain.from_llm(
+            llm=llm,
+            retriever=vectorstore.as_retriever(search_kwargs={"k": 3}),
+            memory=memory,
+            return_source_documents=True,
+            output_key="answer"
+        )
+        return chain
+    except Exception as e:
+        st.error(f"Chatbot creation failed: {str(e)}")
+        return None
+
+def main():
+    st.title("📘 Facebook Public Data Extractor")
+    st.markdown("Extract and analyze public Facebook data - Works on Hugging Face Spaces")
     
     if st.button("← Back to Main Dashboard"):
         st.switch_page("app.py")
     
-    st.warning("""
-    ## ⚠️ Basic Facebook Extractor
-    
-    **For enhanced Facebook features, use Facebook Extractor 2.0**
+    # Initialize session state
+    if "extractor" not in st.session_state:
+        st.session_state.extractor = FacebookPublicExtractor()
+    if "facebook_data" not in st.session_state:
+        st.session_state.facebook_data = None
+    if "vectorstore" not in st.session_state:
+        st.session_state.vectorstore = None
+    if "chatbot" not in st.session_state:
+        st.session_state.chatbot = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
     
-    This version provides basic public data extraction.
+    # Sidebar
+    with st.sidebar:
+        st.header("⚙️ Facebook Configuration")
+        
+        data_type = st.selectbox(
+            "Content Type",
+            ["page", "group", "event", "post", "marketplace"],
+            help="Select the type of Facebook content"
+        )
+        
+        facebook_url = st.text_input(
+            "Facebook URL",
+            placeholder="https://www.facebook.com/Meta/",
+            help="Enter public Facebook URL (pages work best)"
+        )
+        
+        # Suggested URLs that often work
+        st.markdown("### 🚀 Test URLs")
+        test_urls = {
+            "Meta (Facebook)": "https://www.facebook.com/Meta/",
+            "Starbucks": "https://www.facebook.com/Starbucks/",
+            "NASA": "https://www.facebook.com/NASA/",
+        }
+        
+        for name, url in test_urls.items():
+            if st.button(f"🔗 {name}", key=f"fb_{name}"):
+                st.session_state.current_fb_url = url
+                st.rerun()
+        
+        if st.button("🚀 Extract Facebook Data", type="primary"):
+            url_to_use = facebook_url or getattr(st.session_state, 'current_fb_url', '')
+            
+            if not url_to_use:
+                st.error("❌ Please enter a Facebook URL")
+            elif 'facebook.com' not in url_to_use:
+                st.error("❌ Please enter a valid Facebook URL")
+            else:
+                with st.spinner("🔄 Extracting Facebook data..."):
+                    extracted_data = st.session_state.extractor.extract_public_data(url_to_use, data_type)
+                    
+                    if extracted_data.get("status") == "success":
+                        st.session_state.facebook_data = extracted_data
+                        
+                        # Process for AI
+                        vectorstore, chunks = process_facebook_data(extracted_data)
+                        if vectorstore:
+                            st.session_state.vectorstore = vectorstore
+                            st.session_state.chatbot = create_chatbot(vectorstore)
+                            st.session_state.chat_history = []
+                            st.success(f"✅ Extracted {len(extracted_data['content_blocks'])} content blocks!")
+                        else:
+                            st.error("❌ Failed to process data for AI")
+                    else:
+                        error_msg = extracted_data.get("error", "Unknown error")
+                        st.error(f"❌ Extraction failed: {error_msg}")
+        
+        if st.session_state.facebook_data:
+            st.markdown("---")
+            if st.button("🗑️ Clear Data", type="secondary"):
+                st.session_state.facebook_data = None
+                st.session_state.vectorstore = None
+                st.session_state.chatbot = None
+                st.session_state.chat_history = []
+                st.rerun()
     
-    **Features available in Facebook Extractor 2.0:**
-    - Enhanced algorithms
-    - AI-powered analysis  
-    - Better content recognition
-    - Advanced chat capabilities
-    """)
+    # Main content
+    col1, col2 = st.columns([1, 1])
     
-    st.info("""
-    **Click the button below to access the enhanced version:**
-    """)
+    with col1:
+        st.header("📊 Extraction Results")
+        
+        if st.session_state.facebook_data:
+            data = st.session_state.facebook_data
+            page_info = data['page_info']
+            content_blocks = data['content_blocks']
+            
+            st.success("✅ Facebook Data Extracted")
+            
+            # Metrics
+            col1, col2, col3 = st.columns(3)
+            with col1:
+                st.metric("Content Blocks", len(content_blocks))
+            with col2:
+                st.metric("Public Content", sum(1 for b in content_blocks if b['is_public_content']))
+            with col3:
+                st.metric("Response Code", page_info['response_code'])
+            
+            # Page info
+            st.subheader("🏷️ Page Information")
+            st.write(f"**Title:** {page_info['title']}")
+            st.write(f"**URL Type:** {data['url_type']}")
+            st.write(f"**Access:** {page_info['access_note']}")
+            st.write(f"**Extracted:** {data['extraction_time'][:19]}")
+            
+            # Content samples
+            st.subheader("📝 Sample Content")
+            for i, block in enumerate(content_blocks[:3]):
+                with st.expander(f"Block {i+1} - {block['content_type']} ({block['word_count']} words)"):
+                    st.write(block['content'])
+                    st.caption(f"Public: {block['is_public_content']} | Links: {block['has_links']}")
+        
+        else:
+            st.info("""
+            ## 📘 Facebook Public Data Extractor
+            
+            **How it works:**
+            1. Enter a public Facebook URL (pages work best)
+            2. Click "Extract Facebook Data"
+            3. View extracted public content
+            4. Chat with AI about the data
+            
+            **Supported URLs:**
+            - 🏢 Public Facebook Pages
+            - 📘 Public Groups (limited data)
+            - 🎉 Public Events
+            - 📝 Public Posts
+            
+            **Limitations:**
+            - Only public content accessible
+            - No private group data
+            - Limited without login
+            """)
     
-    if st.button("🚀 Switch to Facebook Extractor 2.0", type="primary"):
-        st.switch_page("pages/facebook_extractor_pro.py")
+    with col2:
+        st.header("💬 AI Analysis Chat")
+        
+        if st.session_state.chatbot and st.session_state.facebook_data:
+            # Display chat history
+            for chat in st.session_state.chat_history:
+                if chat["role"] == "user":
+                    with st.chat_message("user"):
+                        st.write(chat['content'])
+                elif chat["role"] == "assistant":
+                    with st.chat_message("assistant"):
+                        st.write(chat['content'])
+            
+            # Chat input
+            user_input = st.chat_input("Ask about the Facebook data...")
+            
+            if user_input:
+                st.session_state.chat_history.append({"role": "user", "content": user_input})
+                
+                with st.spinner("🤔 AI is analyzing..."):
+                    try:
+                        response = st.session_state.chatbot.invoke({"question": user_input})
+                        answer = response.get("answer", "I couldn't generate a response.")
+                        st.session_state.chat_history.append({"role": "assistant", "content": answer})
+                        st.rerun()
+                    except Exception as e:
+                        error_msg = f"AI Error: {str(e)}"
+                        st.session_state.chat_history.append({"role": "assistant", "content": error_msg})
+                        st.rerun()
+            
+            # Suggested questions
+            if not st.session_state.chat_history:
+                st.subheader("💡 Try asking:")
+                suggestions = [
+                    "What is this Facebook page about?",
+                    "Summarize the available public content",
+                    "What type of content is most common?",
+                    "Analyze the page's public information"
+                ]
+                
+                for suggestion in suggestions:
+                    if st.button(suggestion, key=f"fb_suggest_{suggestion}"):
+                        st.info(f"Type: '{suggestion}' in chat")
+        
+        elif st.session_state.facebook_data:
+            st.info("💬 Start chatting with the AI about the Facebook data")
+        else:
+            st.info("🔍 Extract Facebook data to enable AI chat")
 
 if __name__ == "__main__":
     main()