Spaces:
Runtime error
Runtime error
| # Multi-City Global Pilot Data Ingestion & Enrichment Engine | |
| # Pilots: Dubai (UAE) & Istanbul (Turkey) + Amsterdam (NL) | |
| import os | |
| import json | |
| import math | |
| import sys | |
| import random | |
| import pickle | |
| from collections import Counter | |
| import numpy as np | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.decomposition import TruncatedSVD | |
| if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'): | |
| sys.stdout.reconfigure(encoding='utf-8') | |
| DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') | |
| random.seed(42) | |
| def haversine_distance_m(lat1, lon1, lat2, lon2): | |
| R = 6371000.0 | |
| phi1 = math.radians(lat1) | |
| phi2 = math.radians(lat2) | |
| delta_phi = math.radians(lat2 - lat1) | |
| delta_lambda = math.radians(lon2 - lon1) | |
| a = math.sin(delta_phi / 2.0) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2.0) ** 2 | |
| c = 2.0 * math.atan2(math.sqrt(a), math.sqrt(1.0 - a)) | |
| return R * c | |
| def build_semantic_model_for_city(city_id, places): | |
| city_dir = os.path.join(DATA_DIR, city_id) | |
| os.makedirs(city_dir, exist_ok=True) | |
| docs = [] | |
| for p in places: | |
| name = p.get('name', '') | |
| cat = p.get('primary_category', '') | |
| alts = ' '.join(p.get('alternate_categories', [])) | |
| cuisine = p.get('cuisine', '') | |
| badges = ' '.join(p.get('craft_badges', [])) | |
| details = ' '.join(p.get('craft_details', [])) | |
| must_try = p.get('must_try', '') | |
| neighborhood = p.get('neighborhood', '') | |
| address = p.get('address', '') | |
| heritage = p.get('heritage', {}) | |
| heritage_text = (heritage.get('status', '') + ' ' + heritage.get('badge', '') + ' ' + heritage.get('era', '')) if heritage.get('is_monument') else '' | |
| own = p.get('ownership', {}) | |
| own_text = own.get('label', '') + ' ' + own.get('badge', '') | |
| celiac = p.get('celiac_safety', {}) | |
| celiac_text = ('celiac safe gluten free ' + str(celiac.get('badge', '')) + ' ' + str(celiac.get('certified_by', ''))) if celiac.get('is_safe') else '' | |
| scratch = p.get('scratch_kitchen', {}) | |
| scratch_text = ('scratch kitchen in-house production ' + str(scratch.get('badge', ''))) if scratch.get('is_scratch') else '' | |
| hosp = p.get('hospitality', {}) | |
| hosp_text = ('free tap water welcoming hospitality ' + str(hosp.get('badge', ''))) if hosp.get('free_tap_water') else '' | |
| amenities = p.get('amenities', {}) | |
| amenity_text = [] | |
| if amenities.get('outdoor_seating'): amenity_text.append('outdoor seating open air patio terrace') | |
| if amenities.get('canal_terrace'): amenity_text.append('waterfront canal marina terrace view') | |
| if amenities.get('microbrewery'): amenity_text.append('craft microbrewery house brewed') | |
| if amenities.get('in_house_roastery'): amenity_text.append('in house micro roastery specialty coffee beans single origin') | |
| acc = p.get('accreditation') or {} | |
| acc_text = (str(acc.get('award_tier', '')) + ' ' + str(acc.get('review_quote', ''))) if acc else '' | |
| doc = name + '. Type: ' + cat + ' ' + alts + '. Cuisine: ' + cuisine + '. Craft: ' + badges + ' ' + scratch_text + '. Details: ' + details + '. Heritage: ' + heritage_text + '. Ownership: ' + own_text + '. Kitchen Safety: ' + celiac_text + '. Hospitality: ' + hosp_text + '. Amenities: ' + ' '.join(amenity_text) + '. Critical awards: ' + acc_text + '. Location: ' + neighborhood + ' ' + address + '. Must try signature: ' + must_try + '.' | |
| docs.append(doc) | |
| print('Building semantic vector space for ' + city_id + ' with ' + str(len(docs)) + ' places...') | |
| vectorizer = TfidfVectorizer( | |
| ngram_range=(1, 2), | |
| sublinear_tf=True, | |
| max_features=5000, | |
| stop_words='english' | |
| ) | |
| tfidf_matrix = vectorizer.fit_transform(docs) | |
| n_components = min(128, tfidf_matrix.shape[1] - 1) | |
| svd = TruncatedSVD(n_components=n_components, random_state=42) | |
| dense_vectors = svd.fit_transform(tfidf_matrix) | |
| norms = np.linalg.norm(dense_vectors, axis=1, keepdims=True) | |
| norms[norms == 0] = 1.0 | |
| normalized_vectors = dense_vectors / norms | |
| vectors_path = os.path.join(city_dir, 'vectors.npy') | |
| model_path = os.path.join(city_dir, 'semantic_model.pkl') | |
| np.save(vectors_path, normalized_vectors) | |
| with open(model_path, 'wb') as f: | |
| pickle.dump({'vectorizer': vectorizer, 'svd': svd}, f) | |
| print('Saved ' + city_id + ' vectors to ' + vectors_path + ' and model to ' + model_path) | |