# Multi-City Global Pilot Data Ingestion & Enrichment Engine
# Pilots: Dubai (UAE) & Istanbul (Turkey) + Amsterdam (NL)
import os
import json
import math
import sys
import random
import pickle
from collections import Counter
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD

if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'):
    sys.stdout.reconfigure(encoding='utf-8')

DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
random.seed(42)

def haversine_distance_m(lat1, lon1, lat2, lon2):
    R = 6371000.0
    phi1 = math.radians(lat1)
    phi2 = math.radians(lat2)
    delta_phi = math.radians(lat2 - lat1)
    delta_lambda = math.radians(lon2 - lon1)
    a = math.sin(delta_phi / 2.0) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2.0) ** 2
    c = 2.0 * math.atan2(math.sqrt(a), math.sqrt(1.0 - a))
    return R * c

def build_semantic_model_for_city(city_id, places):
    city_dir = os.path.join(DATA_DIR, city_id)
    os.makedirs(city_dir, exist_ok=True)
    docs = []
    for p in places:
        name = p.get('name', '')
        cat = p.get('primary_category', '')
        alts = ' '.join(p.get('alternate_categories', []))
        cuisine = p.get('cuisine', '')
        badges = ' '.join(p.get('craft_badges', []))
        details = ' '.join(p.get('craft_details', []))
        must_try = p.get('must_try', '')
        neighborhood = p.get('neighborhood', '')
        address = p.get('address', '')
        
        heritage = p.get('heritage', {})
        heritage_text = (heritage.get('status', '') + ' ' + heritage.get('badge', '') + ' ' + heritage.get('era', '')) if heritage.get('is_monument') else ''
        
        own = p.get('ownership', {})
        own_text = own.get('label', '') + ' ' + own.get('badge', '')
        
        celiac = p.get('celiac_safety', {})
        celiac_text = ('celiac safe gluten free ' + str(celiac.get('badge', '')) + ' ' + str(celiac.get('certified_by', ''))) if celiac.get('is_safe') else ''
        
        scratch = p.get('scratch_kitchen', {})
        scratch_text = ('scratch kitchen in-house production ' + str(scratch.get('badge', ''))) if scratch.get('is_scratch') else ''
        
        hosp = p.get('hospitality', {})
        hosp_text = ('free tap water welcoming hospitality ' + str(hosp.get('badge', ''))) if hosp.get('free_tap_water') else ''
        
        amenities = p.get('amenities', {})
        amenity_text = []
        if amenities.get('outdoor_seating'): amenity_text.append('outdoor seating open air patio terrace')
        if amenities.get('canal_terrace'): amenity_text.append('waterfront canal marina terrace view')
        if amenities.get('microbrewery'): amenity_text.append('craft microbrewery house brewed')
        if amenities.get('in_house_roastery'): amenity_text.append('in house micro roastery specialty coffee beans single origin')
        
        acc = p.get('accreditation') or {}
        acc_text = (str(acc.get('award_tier', '')) + ' ' + str(acc.get('review_quote', ''))) if acc else ''
        
        doc = name + '. Type: ' + cat + ' ' + alts + '. Cuisine: ' + cuisine + '. Craft: ' + badges + ' ' + scratch_text + '. Details: ' + details + '. Heritage: ' + heritage_text + '. Ownership: ' + own_text + '. Kitchen Safety: ' + celiac_text + '. Hospitality: ' + hosp_text + '. Amenities: ' + ' '.join(amenity_text) + '. Critical awards: ' + acc_text + '. Location: ' + neighborhood + ' ' + address + '. Must try signature: ' + must_try + '.'
        docs.append(doc)

    print('Building semantic vector space for ' + city_id + ' with ' + str(len(docs)) + ' places...')
    vectorizer = TfidfVectorizer(
        ngram_range=(1, 2),
        sublinear_tf=True,
        max_features=5000,
        stop_words='english'
    )
    tfidf_matrix = vectorizer.fit_transform(docs)
    n_components = min(128, tfidf_matrix.shape[1] - 1)
    svd = TruncatedSVD(n_components=n_components, random_state=42)
    dense_vectors = svd.fit_transform(tfidf_matrix)
    norms = np.linalg.norm(dense_vectors, axis=1, keepdims=True)
    norms[norms == 0] = 1.0
    normalized_vectors = dense_vectors / norms

    vectors_path = os.path.join(city_dir, 'vectors.npy')
    model_path = os.path.join(city_dir, 'semantic_model.pkl')
    np.save(vectors_path, normalized_vectors)
    with open(model_path, 'wb') as f:
        pickle.dump({'vectorizer': vectorizer, 'svd': svd}, f)
    print('Saved ' + city_id + ' vectors to ' + vectors_path + ' and model to ' + model_path)