Update train.py

train.py

@@ -1,14 +1,49 @@
-
+"""
+Enhanced Training Script with Online Learning
+Features: age, religion, marital_status, qualification, country, maslak, region_caste,
+          hobbies, personality_traits, preferred_partner_criteria
+Supports: Initial training, Incremental updates, Feature validation
+Saves: encoders, tfidf_vectorizers, scaler, feature matrix, profile data
+"""
 
 import os
+import pickle
 import httpx
+import json
 import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
+from datetime import datetime
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.preprocessing import LabelEncoder, MinMaxScaler
+from sklearn.neighbors import NearestNeighbors
+import time
 
+# Config
 SUPABASE_URL = os.getenv('SUPABASE_URL', 'https://nquhiryqtbrtpauuxmsc.supabase.co')
 SERVICE_KEY = os.getenv('SUPABASE_SERVICE_KEY', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im5xdWhpcnlxdGJydHBhdXV4bXNjIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc3NjA4MzcxMCwiZXhwIjoyMDkxNjU5NzEwfQ.q7ysbaSfMv15g7PyJRLMwnRpEyia5d_ol3iLZn0Xayo')
 HEADERS = {'apikey': SERVICE_KEY, 'Content-Type': 'application/json'}
 
+CATEGORICAL_COLS = ['religion', 'marital_status', 'qualification', 'country', 'maslak', 'region_caste']
+TEXT_COLS = ['hobbies', 'personality_traits', 'preferred_partner_criteria']
+MODEL_PATH = 'models/recommendation_model.pkl'
+TRAINED_IDS_PATH = 'models/trained_profile_ids.json'
+
+
+def fetch_profiles(limit=1000):
+    """Fetch active profiles from Supabase"""
+    url = f"{SUPABASE_URL}/rest/v1/profiles"
+    params = {'select': '*', 'is_active': 'eq.true', 'limit': str(limit)}
+    
+    try:
+        resp = httpx.get(url, headers=HEADERS, params=params, timeout=30)
+        resp.raise_for_status()
+        data = resp.json()
+        print(f"[OK] Fetched {len(data)} profiles from Supabase")
+        return data
+    except Exception as e:
+        print(f"[X] Error fetching profiles: {e}")
+        return []
+
+
 def safe_str(val):
     """Safe string conversion"""
     if val is None:
@@ -18,164 +53,316 @@ def safe_str(val):
     return str(val).strip().lower()
 
 
-async def test_recommend():
-    """Test the recommend flow step by step"""
-    
-    print("\n" + "="*70)
-    print("🔍 DEBUGGING RECOMMEND ENDPOINT")
-    print("="*70)
-    
-    user_id = "ef7f794d-c323-4e5b-9bfa-40bc8d3f5d96"
-    
-    # Step 1: Fetch user
-    print(f"\n1️⃣ Fetching user {user_id}...")
+def preprocess(p):
+    """Convert raw Supabase profile to feature-ready dict"""
     try:
-        resp = httpx.get(
-            f"{SUPABASE_URL}/rest/v1/profiles",
-            headers=HEADERS,
-            params={'id': f'eq.{user_id}', 'select': '*'},
-            timeout=10
-        )
-        print(f"   Status: {resp.status_code}")
-        
-        if resp.status_code != 200:
-            print(f"   ✗ Error fetching user: {resp.status_code}")
-            print(f"   Response: {resp.text}")
-            return
-        
-        users = resp.json()
-        if not users:
-            print(f"   ✗ User not found in database")
-            return
-        
-        user = users[0]
-        print(f"   ✓ User found: {user.get('full_name', 'Unknown')}")
-        print(f"   ✓ Gender: {user.get('gender', 'unknown')}")
-        print(f"   ✓ Age: {user.get('age', 'unknown')}")
-        
-    except Exception as e:
-        print(f"   ✗ Exception: {e}")
-        return
+        age = int(p.get('age') or 25)
+    except:
+        age = 25
+
+    cat = {col: safe_str(p.get(col)) or 'unknown' for col in CATEGORICAL_COLS}
+    text = {col: safe_str(p.get(col)) for col in TEXT_COLS}
+
+    return {
+        'age': age,
+        **cat,
+        **text,
+        'gender': p.get('gender', '').lower(),
+        'is_active': p.get('is_active', True),
+        'id': p['id'],
+        'full_name': p.get('full_name', 'Unknown'),
+        'city': p.get('city', ''),
+        'profile_picture_url': p.get('profile_picture_url')
+    }
+
+
+def load_trained_ids():
+    """Load IDs of profiles already in model"""
+    if os.path.exists(TRAINED_IDS_PATH):
+        try:
+            with open(TRAINED_IDS_PATH, 'r') as f:
+                return set(json.load(f))
+        except:
+            return set()
+    return set()
+
+
+def save_trained_ids(ids):
+    """Save trained profile IDs"""
+    os.makedirs('models', exist_ok=True)
+    with open(TRAINED_IDS_PATH, 'w') as f:
+        json.dump(list(ids), f)
+
+
+def load_model():
+    """Load existing model"""
+    if os.path.exists(MODEL_PATH):
+        try:
+            with open(MODEL_PATH, 'rb') as f:
+                model = pickle.load(f)
+                print(f"[OK] Loaded existing model: {model['n_profiles']} profiles")
+                return model
+        except Exception as e:
+            print(f"[X] Error loading model: {e}")
+    return None
+
+
+def encode_features_initial(profiles):
+    """Initial training: Encode all features"""
+    n = len(profiles)
+
+    # LabelEncode categoricals
+    le_dict = {}
+    cat_matrix = []
+    for col in CATEGORICAL_COLS:
+        le = LabelEncoder()
+        values = [p[col] for p in profiles]
+        le.fit(values)
+        le_dict[col] = le
+        encoded = le.transform(values)
+        cat_matrix.append(encoded.reshape(-1, 1))
+
+    # TF-IDF for text columns
+    tfidf_vectors = []
+    tfidf_dict = {}
+    for col in TEXT_COLS:
+        corpus = [p[col] for p in profiles]
+        tfidf = TfidfVectorizer(max_features=20, stop_words='english', token_pattern=r'(?u)\\b\\w+\\b')
+        vec = tfidf.fit_transform(corpus).toarray()
+        tfidf_dict[col] = tfidf
+        tfidf_vectors.append(vec)
+
+    # Normalize age
+    ages = np.array([[p['age']] for p in profiles], dtype=float)
+    scaler = MinMaxScaler()
+    ages_norm = scaler.fit_transform(ages)
+
+    # Combine: age (1) + cat (6) + tfidf(3*20=60)
+    feature_matrix = np.hstack([ages_norm] + cat_matrix + tfidf_vectors)
     
-    # Step 2: Check gender
-    print(f"\n2️⃣ Validating gender...")
-    user_gender = safe_str(user.get('gender'))
-    print(f"   Gender: {user_gender}")
+    return feature_matrix, le_dict, scaler, tfidf_dict
+
+
+def encode_single_profile(profile, le_dict, scaler, tfidf_dict):
+    """Encode single profile using existing transformers"""
+    features = []
+
+    # Age
+    age_norm = scaler.transform([[float(profile['age'])]])[0]
+    features.extend(age_norm)
+
+    # Categoricals
+    for col in CATEGORICAL_COLS:
+        le = le_dict[col]
+        val = profile[col]
+        try:
+            encoded = le.transform([val])[0:1]
+        except ValueError:
+            if 'unknown' in le.classes_:
+                encoded = le.transform(['unknown'])[0:1]
+            else:
+                encoded = le.transform([le.classes_[0]])[0:1]
+        features.extend(encoded)
+
+    # Text fields
+    for col in TEXT_COLS:
+        tfidf = tfidf_dict[col]
+        vec = tfidf.transform([profile[col]]).toarray()[0]
+        features.extend(vec)
+
+    return np.array(features).reshape(1, -1)
+
+
+def train_initial(profiles):
+    """Initial training on all profiles"""
+    print("\n" + "="*60)
+    print("INITIAL TRAINING")
+    print("="*60)
     
-    if user_gender not in ('male', 'female'):
-        print(f"   ✗ Invalid gender: {user_gender}")
-        return
-    print(f"   ✓ Valid gender")
+    start_time = time.time()
+
+    X, le_dict, scaler, tfidf_dict = encode_features_initial(profiles)
+
+    print(f"\n[OK] Feature encoding completed")
+    print(f"  - Categorical cols: {len(CATEGORICAL_COLS)}")
+    print(f"  - Text cols: {len(TEXT_COLS)}")
+    print(f"  - Feature matrix: {X.shape}")
+
+    n_neighbors = min(10, len(profiles)-1) if len(profiles) > 1 else 1
+    nn = NearestNeighbors(n_neighbors=n_neighbors, metric='cosine', algorithm='brute')
+    nn.fit(X)
+    print(f"[OK] NearestNeighbors trained (n_neighbors={n_neighbors})")
+
+    model = {
+        'version': '2.0',
+        'trained_at': datetime.now().isoformat(),
+        'n_profiles': len(profiles),
+        'feature_matrix': X,
+        'profile_ids': [p['id'] for p in profiles],
+        'profile_data': profiles,
+        'label_encoders': le_dict,
+        'scaler': scaler,
+        'tfidf_vectorizers': tfidf_dict,
+        'nn_model': nn
+    }
+
+    duration = time.time() - start_time
+    print(f"[OK] Training completed in {duration:.2f}s")
     
-    opposite_gender = 'female' if user_gender == 'male' else 'male'
-    print(f"   ✓ Looking for: {opposite_gender}")
+    return model
+
+
+def train_incremental(new_profiles, old_model):
+    """Incremental training: add only new profiles"""
+    print("\n" + "="*60)
+    print("INCREMENTAL UPDATE")
+    print("="*60)
     
-    # Step 3: Fetch all profiles
-    print(f"\n3️⃣ Fetching all profiles...")
+    start_time = time.time()
+
+    old_X = old_model['feature_matrix']
+    old_profiles = old_model['profile_data']
+    old_ids = set(old_model['profile_ids'])
+
+    new_only = [p for p in new_profiles if p['id'] not in old_ids]
+
+    if not new_only:
+        print(f"\n[INFO] No new profiles to add")
+        return old_model, 0
+
+    print(f"\n[OK] Found {len(new_only)} new profiles")
+
+    new_features = []
+    for p in new_only:
+        feat = encode_single_profile(
+            p,
+            old_model['label_encoders'],
+            old_model['scaler'],
+            old_model['tfidf_vectorizers']
+        )
+        new_features.append(feat[0])
+
+    new_X = np.vstack(new_features)
+    combined_X = np.vstack([old_X, new_X])
+    combined_profiles = old_profiles + new_only
+    combined_ids = [p['id'] for p in combined_profiles]
+
+    print(f"[OK] Feature matrix merged")
+    print(f"  - Old: {len(old_profiles)}")
+    print(f"  - New: {len(new_only)}")
+    print(f"  - Total: {len(combined_profiles)}")
+    print(f"  - Matrix: {combined_X.shape}")
+
+    n_neighbors = min(10, len(combined_profiles)-1)
+    nn = NearestNeighbors(n_neighbors=n_neighbors, metric='cosine', algorithm='brute')
+    nn.fit(combined_X)
+    print(f"[OK] NearestNeighbors retrained")
+
+    updated_model = {
+        'version': '2.0',
+        'trained_at': datetime.now().isoformat(),
+        'n_profiles': len(combined_profiles),
+        'feature_matrix': combined_X,
+        'profile_ids': combined_ids,
+        'profile_data': combined_profiles,
+        'label_encoders': old_model['label_encoders'],
+        'scaler': old_model['scaler'],
+        'tfidf_vectorizers': old_model['tfidf_vectorizers'],
+        'nn_model': nn
+    }
+
+    duration = time.time() - start_time
+    print(f"[OK] Update completed in {duration:.2f}s")
+
+    return updated_model, len(new_only)
+
+
+def save_model(model):
+    """Save model to disk"""
+    os.makedirs('models', exist_ok=True)
     try:
-        all_profiles = []
-        offset = 0
+        with open(MODEL_PATH, 'wb') as f:
+            pickle.dump(model, f)
+        print(f"[OK] Model saved to {MODEL_PATH}")
         
-        while offset < 1000:
-            resp = httpx.get(
-                f"{SUPABASE_URL}/rest/v1/profiles",
-                headers=HEADERS,
-                params={
-                    'select': '*',
-                    'is_active': 'eq.true',
-                    'limit': 100,
-                    'offset': offset
-                },
-                timeout=15
-            )
-            
-            if resp.status_code != 200:
-                print(f"   ✗ Error at offset {offset}: {resp.status_code}")
-                break
-            
-            batch = resp.json()
-            if not batch:
-                break
-            
-            all_profiles.extend(batch)
-            offset += 100
-            print(f"   ✓ Fetched {len(all_profiles)} profiles...", end='\r')
-        
-        print(f"\n   ✓ Total profiles: {len(all_profiles)}")
+        save_trained_ids(model['profile_ids'])
+        print(f"[OK] Trained IDs saved")
         
+        return True
     except Exception as e:
-        print(f"   ✗ Exception: {e}")
-        return
+        print(f"[X] Error saving model: {e}")
+        return False
+
+
+def validate_model(model, num_tests=3):
+    """Validate model on sample profiles"""
+    print("\n" + "="*60)
+    print("VALIDATION TEST")
+    print("="*60)
     
-    # Step 4: Filter candidates
-    print(f"\n4️⃣ Filtering candidates (opposite gender, active, not self)...")
-    candidates = []
-    for i, p in enumerate(all_profiles):
-        p_gender = safe_str(p.get('gender'))
-        p_active = p.get('is_active')
-        p_id = p.get('id')
+    profiles = model['profile_data']
+    X = model['feature_matrix']
+    nn = model['nn_model']
+
+    for idx in range(min(num_tests, len(profiles))):
+        test_vec = X[idx:idx+1]
+        distances, indices = nn.kneighbors(test_vec, n_neighbors=min(5, len(profiles)))
+
+        name = profiles[idx]['full_name']
+        print(f"\nTop matches for {name}:")
         
-        if p_gender == opposite_gender and p_active and p_id != user_id:
-            candidates.append((i, p))
-    
-    print(f"   ✓ Found {len(candidates)} candidates")
+        for rank, (dist, match_idx) in enumerate(zip(distances[0], indices[0]), 1):
+            sim_score = (1 - dist) * 100
+            match_name = profiles[match_idx]['full_name']
+            print(f"  {rank}. {match_name} - {sim_score:.1f}%")
+
+
+def main():
+    print("\n" + "="*70)
+    print("MATCHMAKING MODEL TRAINING")
+    print("="*70)
+
+    print("\n[1] Fetching profiles from Supabase...")
+    raw_profiles = fetch_profiles()
     
-    if len(candidates) < 1:
-        print(f"   ✗ No candidates found!")
+    if not raw_profiles:
+        print("[X] No profiles fetched. Exiting.")
         return
     
-    # Step 5: Check profile data quality
-    print(f"\n5️⃣ Checking profile data quality...")
-    
-    # Check user profile
-    print(f"\n   User profile fields:")
-    for key in ['age', 'religion', 'marital_status', 'qualification', 'country', 'maslak', 'region_caste', 'hobbies', 'personality_traits', 'preferred_partner_criteria']:
-        val = user.get(key)
-        print(f"     {key:30} = {str(val)[:50]}")
-    
-    # Check first candidate
-    if candidates:
-        print(f"\n   First candidate profile fields:")
-        cand = candidates[0][1]
-        for key in ['age', 'religion', 'marital_status', 'qualification', 'country', 'maslak', 'region_caste', 'hobbies', 'personality_traits', 'preferred_partner_criteria']:
-            val = cand.get(key)
-            print(f"     {key:30} = {str(val)[:50]}")
-    
-    # Step 6: Test feature encoding
-    print(f"\n6️⃣ Testing feature encoding...")
-    
-    CATEGORICAL_COLS = ['religion', 'marital_status', 'qualification', 'country', 'maslak', 'region_caste']
-    TEXT_COLS = ['hobbies', 'personality_traits', 'preferred_partner_criteria']
-    
-    try:
-        # Age
-        age = int(user.get('age') or 25)
-        print(f"   ✓ Age: {age}")
-        
-        # Categorical
-        for col in CATEGORICAL_COLS:
-            val = safe_str(user.get(col)) or 'unknown'
-            print(f"   ✓ {col}: {val}")
-        
-        # Text
-        for col in TEXT_COLS:
-            val = safe_str(user.get(col))
-            print(f"   ✓ {col}: {val[:50] if val else '(empty)'}...")
-        
-    except Exception as e:
-        print(f"   ✗ Encoding error: {e}")
-        import traceback
-        traceback.print_exc()
+    if len(raw_profiles) < 10:
+        print("[X] Need at least 10 profiles. Got " + str(len(raw_profiles)))
         return
-    
-    print(f"\n✅ All checks passed!")
-    print(f"\nThe recommend endpoint should work. Check:")
-    print(f"1. Logs in HF Space for detailed error")
-    print(f"2. Verify SUPABASE_SERVICE_KEY is valid")
-    print(f"3. Check /health endpoint for cache status")
-    print(f"4. Try POST /refresh-cache to force update")
+
+    print("[2] Preprocessing profiles...")
+    profiles = [preprocess(p) for p in raw_profiles]
+    print("[OK] Preprocessed " + str(len(profiles)) + " profiles")
+
+    existing_model = load_model()
+
+    if existing_model is None:
+        model = train_initial(profiles)
+    else:
+        model, new_count = train_incremental(profiles, existing_model)
+        if new_count == 0:
+            print("\n[INFO] No updates needed. Model is current.")
+            return
+
+    print("\n[3] Saving model...")
+    if not save_model(model):
+        print("[X] Failed to save model")
+        return
+
+    validate_model(model)
+
+    print("\n" + "="*70)
+    print("TRAINING COMPLETED SUCCESSFULLY!")
+    print("="*70)
+    print("\nSummary:")
+    print("  - Total profiles: " + str(model['n_profiles']))
+    print("  - Feature dimensions: " + str(model['feature_matrix'].shape[1]))
+    print("  - Model version: " + model['version'])
+    print("  - Trained at: " + model['trained_at'])
 
 
 if __name__ == '__main__':
-    import asyncio
-    asyncio.run(test_recommend())
+    main()