Spaces:
Sleeping
Sleeping
File size: 2,948 Bytes
fddb26d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import pandas as pd
import numpy as np
from sentence_transformers import SentenceTransformer
import os
# ==============================================================================
# SCRIPT: embedding.py
# DESCRIPTION: Converts text data into semantic vectors (embeddings).
# ==============================================================================
def generate_embeddings(input_path, output_metadata_path, output_vector_path):
# 1. Data Loading
if not os.path.exists(input_path):
print(f"Error: {input_path} not found!")
return
print("Reading normalized POI data...")
df = pd.read_json(input_path)
# Validate that the necessary column exists for the model
if 'enriched_description' not in df.columns:
print("Error: 'enriched_description' column missing. Check the normalization step.")
return
# 2. Model Initialization (multilingual-e5-small)
# This model is optimized for both Turkish and English semantic search.
print("Loading model: multilingual-e5-small...")
model = SentenceTransformer('intfloat/multilingual-e5-small')
# 3. Text Preprocessing
# The E5 model family requires the 'passage: ' prefix for documents to optimize retrieval.
print("Preparing texts for vectorization...")
texts = ["passage: " + str(text) for text in df['enriched_description']]
# 4. Embedding Generation
print(f"Computing embeddings for {len(texts)} locations... (This may take a moment)")
# convert_to_numpy=True ensures the output is ready for binary storage and math operations.
embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True)
# 5. Modular Storage
# A. Vector File (.npy): Stored in binary format for high-speed retrieval and low memory footprint.
np.save(output_vector_path, embeddings)
print(f"Vectors saved to: {output_vector_path}")
# B. Metadata File (.json): Saves the original data without the heavy vector arrays.
df_metadata = df.drop(columns=['embedding'], errors='ignore')
df_metadata.to_json(output_metadata_path, orient="records", force_ascii=False, indent=4)
print(f"Metadata saved to: {output_metadata_path}")
print("\n Embedding process completed successfully!")
if __name__ == "__main__":
# Get the absolute path of the directory where this script is located
script_dir = os.path.dirname(os.path.abspath(__file__))
# Navigate to the 'app' directory (one level up from 'scripts')
app_dir = os.path.dirname(script_dir)
# Define the data directory path
data_dir = os.path.join(app_dir, "data")
# Define file paths relative to the data directory
INPUT_FILE = os.path.join(data_dir, "data_sightseeing_ready.json")
META_OUTPUT = os.path.join(data_dir, "poi_metadata.json")
VEC_OUTPUT = os.path.join(data_dir, "poi_vectors.npy")
generate_embeddings(INPUT_FILE, META_OUTPUT, VEC_OUTPUT)
|