TestSpace / processing.py
AntonyManojXavier's picture
Update processing.py
7a66e23 verified
Raw History Blame Contribute Delete
6.74 kB
import os
import dgl
import torch
import pandas as pd
import ast
import numpy as np
import joblib
import json
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from models import ModifiedGCN
# Function to load and clean JSON data
def load_and_clean_json_data(file):
def clean_value(value):
try:
if '.' in value:
return float(value)
else:
return int(value)
except ValueError:
return value
with open(file, 'r') as f:
data = json.load(f)
for entry in data:
for key, value in entry.items():
if isinstance(value, str) and value.replace('.', '', 1).replace('-', '', 1).isdigit():
entry[key] = clean_value(value)
elif isinstance(value, str) and (value.startswith('[') and value.endswith(']')):
entry[key] = ast.literal_eval(value)
return data
# Function to process the file and return status and processed file
def process_file(file):
if file is not None:
status = "Processing"
scaler_path = 'label_scaler.pkl'
label_scaler = joblib.load(scaler_path)
graphs = {}
try:
site_data = load_and_clean_json_data(file)
except Exception as e:
return f"Error loading JSON data: {e}", None
combination_name = file.name.replace('.json', '')
src_nodes = []
dst_nodes = []
node_features = []
edge_features = []
edge_weights = []
edge_set = set()
for site in site_data:
try:
site['children_id'] = ast.literal_eval(str(site['children_id']))
site['angle_node_children'] = ast.literal_eval(str(site['angle_node_children']))
site['edge_length'] = ast.literal_eval(str(site['edge_length']))
except Exception as e:
return f"Error processing site data: {e}", None
node_features.append([
site['num_children'], site['x_coordinate'], site['y_coordinate'], site['z_coordinate'],
site['area'], site['volume'], site['avg_normals_x'],
site['avg_normals_y'], site['avg_normals_z'], site['bb_depth'],
site['bb_width'], site['bb_height'], site['surface_area'], site['density'], site['vertices_faces_ratio'], site['vertices_edges_ratio']
])
src_node = site['node_id']
children_ids = site['children_id']
angle_node_children = site['angle_node_children']
edge_length = site['edge_length']
for child_id, angle, length in zip(children_ids, angle_node_children, edge_length):
edge = (src_node, child_id)
reverse_edge = (child_id, src_node)
if edge not in edge_set and reverse_edge not in edge_set:
src_nodes.append(src_node)
dst_nodes.append(child_id)
edge_features.append([angle, length])
edge_weights.append(length) # Use edge length as weight
edge_set.add(edge)
edge_set.add(reverse_edge) # Ensure reverse edge is also added
# Create graph
g = dgl.graph((src_nodes, dst_nodes))
# Add node features
node_features_tensor = torch.tensor(node_features, dtype=torch.float32)
# Append meteorological data to each node's features
meteorological_data = [
site_data[0]['prevailing_wind'], site_data[0]['avg_wind_m/s'], site_data[0]['yearly_overall_sun_h'], site_data[0]['yearly_overall_radiation_wh/sqm'], site_data[0]['yearly_avg_temp'], site_data[0]['min_temp'], site_data[0]['max_temp']
]
meteo_tensor = torch.tensor(meteorological_data, dtype=torch.float32).view(1, -1)
meteo_tensor = meteo_tensor.repeat(g.num_nodes(), 1) # Repeat for each node
combined_features_tensor = torch.cat((node_features_tensor, meteo_tensor), dim=1) # Concatenate feature-wise
# Apply bipolar log transformation
combined_features_log = torch.sign(combined_features_tensor) * torch.log1p(torch.abs(combined_features_tensor))
g.ndata['feature'] = combined_features_log
# Add edge features
g.edata['feature'] = torch.tensor(edge_features, dtype=torch.float32)
# Add edge weights
g.edata['weight'] = torch.tensor(edge_weights, dtype=torch.float32)
graphs[combination_name] = g
print(f'Processed graph for combination {combination_name}: {g.num_nodes()} nodes, {g.num_edges()} edges.')
try:
# Load the model with the custom class
model_path = 'complete_model.pth'
model = torch.load(model_path, map_location=torch.device('cpu'))
except Exception as e:
return f"Error loading model: {e}", None
# Define new_graphs using graphs dictionary values
new_graphs = [dgl.add_self_loop(g) for g in graphs.values()]
new_batch = dgl.batch(new_graphs)
new_features = new_batch.ndata['feature']
new_edge_features = new_batch.edata['feature']
new_edge_weights = new_batch.edata['weight']
model.eval()
with torch.no_grad():
predictions = model(new_batch, new_features, new_edge_features, new_edge_weights)
predictions = label_scaler.inverse_transform(predictions.numpy())
# Extract combination name
combination_name = os.path.splitext(os.path.basename(file.name))[0]
# Define column names for the predictions
columns = [
'avg_wind_around_combination_m/s', 'area_percentage_low_wind', 'avg_radiation_walls_wh/sqm',
'area_percentage_high_radiation_wh/sqm', 'overall_radiation_walls_wh/sqm', 'area_percentage_visibility>50%',
'egress_distance_sum_shortest_paths_m', 'egress_distance_num_paths>60m', 'yearly_energy_consumption_kWh'
]
# Convert predictions to list of dicts and ensure all values are float
predictions_list = []
for pred in predictions:
pred_dict = {col: float(pred_val) for col, pred_val in zip(columns, pred)}
predictions_list.append(pred_dict)
# Create a dictionary with combination name and predictions
data = {
'combination_name': combination_name,
'predictions': predictions_list
}
# Save the data to a JSON file
output_file = 'predictions.json'
with open(output_file, 'w') as f:
json.dump(data, f, indent=4)
status = "Finished"
return status, output_file
return "No file uploaded.", None