Cleaned up some files

database_maintenance/pruner_2.py

@@ -1,5 +1,6 @@
 from tqdm import tqdm
 from supabase import create_client, Client
+import os
 
 # --- Configuration ---
 SECRETS_PATH = "secrets.json"  # Adjust path if needed

modelUpdating/bulkLabelByCentroid_2.py

@@ -1,112 +1,3 @@
-# import chromadb
-# from bertopic import BERTopic
-# import numpy as np
-# import time
-# import os
-# import supabase
-
-# # --- CONFIGURATION ---
-# CHROMA_PATH = "/home/wyomike/topicBuzz/my_mastodon_db"
-# COLLECTION_NAME = "mastodon_posts"
-# MODEL_PATH = "my_mastodon_model_reduced" # Using the clean/reduced model
-# BATCH_SIZE = 2000  # Conservative batch size for read/write operations
-
-# # --- CONFIGURATION ---
-# SUPABASE_URL = os.environ.get("SUPABASE_URL") # Ensure these are in your .bashrc or .env
-# SUPABASE_KEY = os.environ.get("SUPABASE_KEY") 
-# LABELS_PATH = "topic_labels.json"
-# MODEL_FILE = "my_online_mastodon_model.pkl"
-# BUCKET = "topic_assets"
-
-# EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2" 
-
-# def load_remote_model():
-#     try:
-#         print("☁️ Downloading model from Supabase Storage...")
-#         with open(MODEL_FILE, 'wb') as f:
-#             res = supabase.storage.from_(BUCKET).download(MODEL_FILE)
-#             f.write(res)
-#         return BERTopic.load(MODEL_FILE)
-#     except Exception as e:
-#         print(f"⚠️ Could not load remote model (first run?): {e}")
-#         return None
-
-# # --- SETUP ---
-# print(f"Connecting to ChromaDB at {CHROMA_PATH}...")
-# client = chromadb.PersistentClient(path=CHROMA_PATH)
-# collection = client.get_collection(COLLECTION_NAME)
-
-# print(f"Loading BERTopic model from {MODEL_PATH}...")
-# topic_model = BERTopic.load(MODEL_PATH)
-
-# # Get total count to track progress
-# total_docs = collection.count()
-# print(f"Total documents to process: {total_docs}")
-
-# # --- MAIN LOOP ---
-# print("Starting bulk topic assignment...")
-# start_time = time.time()
-
-# for offset in range(0, total_docs, BATCH_SIZE):
-#     try:
-#         # 1. Fetch Batch (Include metadatas so we don't overwrite/lose existing ones!)
-#         batch_data = collection.get(
-#             limit=BATCH_SIZE,
-#             offset=offset,
-#             include=["documents", "embeddings", "metadatas"]
-#         )
-        
-#         ids = batch_data['ids']
-#         docs = batch_data['documents']
-#         embeddings = batch_data['embeddings']
-#         metadatas = batch_data['metadatas']
-        
-#         if not ids:
-#             break
-
-#         # 2. Predict Topics
-#         # We pass embeddings to speed this up massively (skips re-calculating them)
-#         # Note: We convert to numpy array as BERTopic expects it
-#         topics, _ = topic_model.transform(docs, np.array(embeddings))
-        
-#         # 3. Prepare Metadata Updates
-#         updated_metadatas = []
-        
-#         for i, topic_id in enumerate(topics):
-#             # Get existing metadata or create empty dict if None
-#             meta = metadatas[i] if metadatas[i] is not None else {}
-            
-#             # Add the new cluster ID
-#             # We convert to int just to be safe for JSON serialization
-#             meta['cluster_id'] = int(topic_id)
-            
-#             # OPTIONAL: Add the topic label string if you want it readable in DB
-#             meta['cluster_label'] = topic_model.get_topic_info(topic_id).Name.values[0]
-            
-#             updated_metadatas.append(meta)
-
-#         # 4. Update ChromaDB
-#         # We ONLY update metadatas. Documents and embeddings remain untouched.
-#         collection.update(
-#             ids=ids,
-#             metadatas=updated_metadatas
-#         )
-        
-#         # Progress Log
-#         processed_count = offset + len(ids)
-#         percent = (processed_count / total_docs) * 100
-#         elapsed = time.time() - start_time
-#         print(f"[{percent:.1f}%] Processed {processed_count}/{total_docs} docs... (Time: {elapsed:.0f}s)")
-
-#     except Exception as e:
-#         print(f"❌ Error processing batch starting at offset {offset}: {e}")
-#         # Optional: Decide if you want to 'continue' or 'break' here
-#         # continue 
-
-# print("✅ Done! All documents have been assigned a 'cluster_id'.")
-
-
-
 import os
 import time
 import numpy as np

modelUpdating/modelUpdater_1.py

@@ -1,238 +1,3 @@
-# import os
-# import json
-# import shutil
-# import numpy as np
-# import datetime
-# from supabase import create_client, Client
-# from bertopic import BERTopic
-# from sentence_transformers import SentenceTransformer
-# from sklearn.cluster import MiniBatchKMeans
-# from sklearn.decomposition import IncrementalPCA
-# from bertopic.vectorizers import OnlineCountVectorizer
-# import gzip
-
-# # --- CONFIGURATION ---
-# SUPABASE_URL = os.environ.get("SUPABASE_URL") # Ensure these are in your .bashrc or .env
-# SUPABASE_KEY = os.environ.get("SUPABASE_KEY") 
-
-# # MODEL_PATH = "my_online_mastodon_model.pkl"
-# LABELS_PATH = "topic_labels.json"
-# STATE_FILE = "training_state.json" # Tracks the last timestamp we processed
-
-# MODEL_FILE = "my_online_mastodon_model.pkl"
-# COMPRESSED_FILE = "my_online_mastodon_model.pkl.gz"  # New filename for cloud
-# BUCKET = "topic_assets"
-
-# # IMPORTANT: This MUST match the model used to create your original embeddings!
-# EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2" 
-
-
-# def load_remote_model():
-#     # Check if we already have the raw model
-#     if os.path.exists(MODEL_FILE):
-#         return BERTopic.load(MODEL_FILE)
-
-#     print(f"☁️ Downloading {COMPRESSED_FILE} from Supabase...")
-#     try:
-#         # 1. Download the compressed file
-#         with open(COMPRESSED_FILE, 'wb') as f:
-#             res = supabase.storage.from_(BUCKET).download(COMPRESSED_FILE)
-#             f.write(res)
-        
-#         # 2. Decompress it
-#         print("   -> Decompressing...")
-#         with gzip.open(COMPRESSED_FILE, 'rb') as f_in:
-#             with open(MODEL_FILE, 'wb') as f_out:
-#                 shutil.copyfileobj(f_in, f_out)
-        
-#         # 3. Load the raw model
-#         return BERTopic.load(MODEL_FILE)
-
-#     except Exception as e:
-#         print(f"⚠️ Could not load remote model: {e}")
-#         return None
-
-# def save_remote_model(model):
-#     print("💾 Saving and compressing model...")
-#     # 1. Save locally as raw pickle
-#     model.save(MODEL_FILE, serialization="pickle")
-    
-#     # 2. Compress it (Raw 95MB -> Compressed ~30MB)
-#     with open(MODEL_FILE, 'rb') as f_in:
-#         with gzip.open(COMPRESSED_FILE, 'wb') as f_out:
-#             shutil.copyfileobj(f_in, f_out)
-    
-#     # 3. Upload the COMPRESSED file
-#     print(f"☁️ Uploading {COMPRESSED_FILE} to Supabase...")
-#     with open(COMPRESSED_FILE, 'rb') as f:
-#         supabase.storage.from_(BUCKET).upload(
-#             path=COMPRESSED_FILE, 
-#             file=f, 
-#             file_options={"x-upsert": "true"}
-#         )
-    
-#     # Clean up local files to save space
-#     if os.path.exists(MODEL_FILE): os.remove(MODEL_FILE)
-#     if os.path.exists(COMPRESSED_FILE): os.remove(COMPRESSED_FILE)
-
-
-# # --- 1. Setup Connections ---
-# if not SUPABASE_URL or not SUPABASE_KEY:
-#     raise ValueError("Missing Supabase credentials in environment variables.")
-
-# supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
-# embedding_model = SentenceTransformer(EMBEDDING_MODEL_NAME)
-
-# # --- 2. Load State (Incremental Logic) ---
-# last_processed_ts = 0
-# if os.path.exists(STATE_FILE):
-#     with open(STATE_FILE, 'r') as f:
-#         state = json.load(f)
-#         last_processed_ts = state.get('last_timestamp', 0)
-
-# print(f"📅 Looking for posts newer than timestamp: {last_processed_ts}")
-
-
-# print("📥 Checking for previous state in cloud...")
-# last_processed_ts = 0
-
-# try:
-#     # Try to download state file from Supabase
-#     with open(STATE_FILE, 'wb') as f:
-#         res = supabase.storage.from_(BUCKET).download(STATE_FILE)
-#         f.write(res)
-    
-#     # If successful, read it
-#     with open(STATE_FILE, 'r') as f:
-#         state = json.load(f)
-#         last_processed_ts = state.get('last_timestamp', 0)
-#     print(f"   -> Resuming from timestamp: {last_processed_ts}")
-
-# except Exception as e:
-#     print(f"   -> No remote state found (or error: {e}). Starting from 0.")
-#     last_processed_ts = 0
-
-
-# # --- 3. Load or Initialize BERTopic ---
-# # if os.path.exists(MODEL_PATH):
-# print(f"🔄 Loading existing model from supabase...")
-# topic_model = load_remote_model()
-# if topic_model is None:
-#     print("✨ Creating NEW model structure...")
-#     umap_model = IncrementalPCA(n_components=5)
-#     cluster_model = MiniBatchKMeans(n_clusters=200, random_state=42)
-#     vectorizer_model = OnlineCountVectorizer(stop_words="english", min_df=10)
-    
-#     topic_model = BERTopic(
-#         embedding_model=None, # We pass embeddings manually
-#         umap_model=umap_model,
-#         hdbscan_model=cluster_model,
-#         vectorizer_model=vectorizer_model
-#     )    
-
-# # --- 4. Fetch Data from Supabase ---
-# BATCH_SIZE = 1000
-# total_processed = 0
-# new_max_ts = last_processed_ts
-
-# while True:
-#     # Fetch batch of new posts
-#     # We select 'content' and 'post_timestamp'. Adjust column names if yours differ.
-#     try:
-#         response = supabase.table("posts") \
-#             .select("content, post_timestamp") \
-#             .gt("post_timestamp", last_processed_ts) \
-#             .order("post_timestamp", desc=False) \
-#             .limit(BATCH_SIZE) \
-#             .execute()
-        
-#         posts = response.data
-#     except Exception as e:
-#         print(f"❌ Supabase Error: {e}")
-#         break
-
-#     if not posts:
-#         print("✅ No more new data to process.")
-#         break
-
-#     # Extract text and update timestamp tracker
-#     batch_docs = [p['content'] for p in posts if p['content']]
-#     batch_timestamps = [p['post_timestamp'] for p in posts]
-    
-#     if not batch_docs:
-#         continue
-
-#     # Update our cursor for the next batch
-#     current_batch_max = max(batch_timestamps)
-#     last_processed_ts = current_batch_max
-#     new_max_ts = max(new_max_ts, current_batch_max)
-
-#     print(f"🧠 Generating embeddings for {len(batch_docs)} posts...")
-    
-#     # Generate Embeddings locally
-#     embeddings = embedding_model.encode(batch_docs, show_progress_bar=False)
-    
-#     # Train the Model
-#     topic_model.partial_fit(batch_docs, np.array(embeddings).astype(np.float64))
-    
-#     total_processed += len(batch_docs)
-#     print(f"   -> Processed batch. Total new: {total_processed}")
-
-# if total_processed > 0:
-#     print(f"💾 Saving updated assets to Supabase (Bucket: {BUCKET})...")
-
-#     # ---------------------------
-#     # A. Regenerate Labels Map
-#     # ---------------------------
-#     labels_map = {}
-#     info_df = topic_model.get_topic_info()
-#     for index, row in info_df.iterrows():
-#         topic_id = row['Topic']
-#         # Handle outlier topic -1 (JSON keys must be strings)
-#         if topic_id == -1:
-#             labels_map[f"Topic_{topic_id}"] = "Outliers"
-#         else:
-#             labels_map[f"Topic_{topic_id}"] = row['Name']
-
-#     # ---------------------------
-#     # B. Upload All Artifacts
-#     # ---------------------------
-
-#     # 1. Upload Model (using the helper function defined earlier)
-#     save_remote_model(topic_model)
-
-#     # 2. Upload Labels
-#     print("   -> Uploading labels...")
-#     with open(LABELS_PATH, "w") as f:
-#         json.dump(labels_map, f)
-    
-#     with open(LABELS_PATH, "rb") as f:
-#         supabase.storage.from_(BUCKET).upload(
-#             LABELS_PATH, 
-#             f, 
-#             file_options={"x-upsert": "true"}
-#         )
-
-#     # 3. Upload State (Crucial for Hugging Face!)
-#     # We must save the new timestamp to the cloud, or the bot will forget it processed these posts.
-#     print(f"   -> Updating state to timestamp {new_max_ts}...")
-#     state_data = {"last_timestamp": new_max_ts}
-    
-#     with open(STATE_FILE, "w") as f:
-#         json.dump(state_data, f)
-
-#     with open(STATE_FILE, "rb") as f:
-#         supabase.storage.from_(BUCKET).upload(
-#             STATE_FILE, 
-#             f, 
-#             file_options={"x-upsert": "true"}
-#         )
-
-#     print(f"✅ Success! Trained on {total_processed} new posts and synced to cloud.")
-
-# else:
-#     print("💤 No new data found. Model untouched.")
-
 import os
 import json
 import shutil

modelUpdating/naturalLanguageTopics_3.py

@@ -1,145 +1,3 @@
-# import google.genai as genai
-# from bertopic import BERTopic
-# import json
-# import os
-# import time
-# from tqdm import tqdm
-
-# NSFW_KEYWORDS = {
-#     "nsfw", "porn", "nude", "naked", "sex", "18+", "xxx", 
-#     "cum", "dick", "pussy", "hentai", "erotic", "onlyfans",
-#     "boobs", "tits", "cock", "slut", "whore", "bitch", "incest",
-#     "lewd", "horny", "sensual", "fetish", "bondage"
-# }
-
-# def get_safe_topic_ids(model, nsfw_set):
-#     """
-#     Returns a list of Topic IDs that do NOT contain any blocklisted words
-#     in their top 10 keywords.
-#     """
-#     safe_ids = []
-#     unsafe_ids = []
-    
-#     # get_topics() returns a dictionary: {topic_id: [(word, score), ...]}
-#     all_topics = model.get_topics()
-    
-#     print(f"Scanning {len(all_topics)} topics for NSFW content...")
-    
-#     for topic_id, words_with_scores in all_topics.items():
-#         # Topic -1 is outliers. We usually keep it, but check its content too.
-        
-#         # Extract just the words from the tuples
-#         topic_words = {word for word, score in words_with_scores}
-        
-#         # Check for intersection
-#         # If the intersection is NOT empty, it contains a bad word
-#         intersection = topic_words.intersection(nsfw_set)
-        
-#         if intersection:
-#             print(f"  [BLOCKED] Topic {topic_id}")
-#             unsafe_ids.append(topic_id)
-#         else:
-#             safe_ids.append(topic_id)
-            
-#     print(f"\nResult: {len(safe_ids)} Safe, {len(unsafe_ids)} Unsafe.")
-#     return safe_ids
-
-# # --- CONFIGURATION ---
-# MODEL_PATH = "my_mastodon_model_reduced"
-# OUTPUT_JSON = "topic_labels.json"
-# API_KEY = json.load(open("config.json"))["googleKey"] # Replace or set env var
-
-# # --- SETUP ---
-# client = genai.Client(api_key=API_KEY)
-# # model = client.models.get() # Fast and cheap
-
-# def generate_labels():
-#     if not os.path.exists(MODEL_PATH):
-#         print("Model not found!")
-#         return
-
-#     print("Loading BERTopic model...")
-#     topic_model = BERTopic.load(MODEL_PATH)
-    
-#     # Get the master table of topic info
-#     info = topic_model.get_topic_info()
-#     print(f"Found {len(info)} topics.")
-#     safe_topics = get_safe_topic_ids(topic_model, nsfw_set=NSFW_KEYWORDS)
-    
-#     # Dictionary to map "Topic_5" -> "Nature Photography"
-#     labels_map = {}
-
-#     print("Asking Gemini to label topics...")
-    
-#     for index, row in tqdm(info.iterrows(), total=len(info)):
-#         if row['Topic'] not in safe_topics:
-#             labels_map[f"Topic_{row['Topic']}"] = "NSFW"  # TODO - check if this is actually the bag of words
-#             continue
-#         topic_id = row['Topic']
-        
-#         # Handle Outliers
-#         if topic_id == 0:
-#             labels_map[f"Topic_{topic_id}"] = "Outliers / Noise"
-#             continue
-
-#         # 1. Get Bag of Words
-#         # row['Representation'] is a list of keywords like ['cat', 'meow', 'feline']
-#         keywords = row['Representation']
-        
-#         # 2. Get Representative Docs
-#         # row['Representative_Docs'] is a list of actual posts
-#         docs = row['Representative_Docs']
-        
-#         # 3. Construct Prompt
-#         # We take top 10 keywords and top 3 docs to save tokens
-#         prompt = f"""
-#         I have a cluster of social media posts from Mastodon.
-        
-#         Keywords: {', '.join(keywords[:10])}
-        
-#         Representative Posts:
-#         {chr(10).join([f"- {d[:300]}..." for d in docs[:3]])}
-        
-#         Based on this, provide a short, specific, natural language label (max 5 words) for this topic.
-#         Do not use quotes. Just the label.
-#         """
-        
-#         try:
-#             # 4. Call Gemini
-#             response = client.models.generate_content(
-#                 model='gemini-2.5-flash',
-#                 contents=prompt
-#             )
-#             label = response.text.strip()
-            
-#             # 5. Store result
-#             labels_map[f"Topic_{topic_id}"] = label
-            
-#             # Sleep briefly to respect rate limits if needed
-#             time.sleep(1.0)
-            
-#         except Exception as e:
-#             print(f"Error labeling Topic {topic_id}: {e}")
-#             labels_map[f"Topic_{topic_id}"] = f"Topic {topic_id}"
-
-#     # Save to JSON
-#     with open("topic_labels.json", "w") as f:
-#         json.dump(labels_map, f)
-
-#     with open("topic_labels.json", "rb") as f:
-#         supabase.storage.from_("topic_assets").upload(
-#             "topic_labels.json", 
-#             f, 
-#             file_options={"x-upsert": "true"}
-#         )
-#     print("☁️ Labels uploaded to cloud.")
-        
-#     print(f"Success! Labels saved to {OUTPUT_JSON}")
-
-# if __name__ == "__main__":
-#     generate_labels()
-
-
 import os
 import json
 import time

pages/TopicWeb.py

@@ -1,6 +1,3 @@
-
-
-
 import streamlit as st
 import networkx as nx
 import pickle