Spaces:
Sleeping
Sleeping
File size: 1,833 Bytes
33306b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | import pandas as pd
import torch
from sentence_transformers import SentenceTransformer
import numpy as np
def create_embeddings(
input_file: str,
output_file: str,
model_name: str = "sentence-transformers/all-MiniLM-L6-v2"
):
"""
Generate and save embeddings using SentenceTransformer.
Args:
input_file: Path to input CSV containing Title and Description columns
output_file: Path to save embeddings
model_name: Name of the sentence-transformer model to use
"""
print("Loading data...")
df = pd.read_csv(input_file)
print(f"Loading model: {model_name}")
embedder = SentenceTransformer(model_name)
# Use GPU if available
if not torch.cuda.is_available():
print("Warning: No GPU found. Using CPU.")
else:
print("GPU found! Using CUDA.")
embedder = embedder.to('cuda')
# Generate combined strings for embedding
print("Preparing text for embedding...")
descriptions = df.apply(
lambda row: f"Title: {row['Title']}; Description: {row['Description']}",
axis=1
).tolist()
# Generate embeddings
print("Generating embeddings...")
embeddings = embedder.encode(
descriptions,
show_progress_bar=True,
batch_size=32 if torch.cuda.is_available() else 8
)
# Save embeddings
print("Saving embeddings...")
embeddings_df = pd.DataFrame(
embeddings,
columns=[f"dim_{i}" for i in range(embeddings.shape[1])]
)
embeddings_df.to_csv(output_file, index=False)
print(f"Embeddings saved to {output_file}")
print(f"Shape of embeddings: {embeddings.shape}")
if __name__ == "__main__":
create_embeddings(
input_file="packages_with_titles.csv",
output_file="embeddings.csv"
) |