from .MyUtils import color_print, pyg_data_to_dgl_graph import torch import dgl from dgl.data import FraudAmazonDataset, FraudYelpDataset import torch_geometric from torch_geometric.transforms import GDC from torch_geometric.data import Data, Dataset, DataLoader from torch_geometric.utils import to_dense_adj, dense_to_sparse from sklearn.model_selection import train_test_split from tqdm import tqdm def splitDataset(index, label, train_ratio): idx_train, idx_rest, y_train, y_rest = train_test_split(index, label[index], stratify=label[index], train_size=train_ratio, random_state=2, shuffle=True) idx_valid, idx_test, y_valid, y_test = train_test_split(idx_rest, y_rest, stratify=y_rest, test_size=0.5, random_state=2, shuffle=True) train_mask = torch.zeros([len(label)]).bool() val_mask = torch.zeros([len(label)]).bool() test_mask = torch.zeros([len(label)]).bool() train_mask[idx_train] = 1 val_mask[idx_valid] = 1 test_mask[idx_test] = 1 return train_mask, val_mask, test_mask def nodeSelect(graph_pyg, nodes_per_subgraph): num = graph_pyg.x.shape[0] num_selected = num - num % nodes_per_subgraph if num_selected: feat_selected=graph_pyg.x[:num_selected] label_selected=graph_pyg.y[:num_selected] train_mask_selected=graph_pyg.train_mask[:num_selected] test_mask_selected=graph_pyg.test_mask[:num_selected] val_mask_selected=graph_pyg.val_mask[:num_selected] edge_index_selected_mask=(graph_pyg.edge_index[0] < num_selected) & (graph_pyg.edge_index[1] < num_selected) edge_index_unselected=graph_pyg.edge_index[:,~edge_index_selected_mask] edge_index_selected=graph_pyg.edge_index[:,edge_index_selected_mask] graph_pyg_selected=Data(x=feat_selected, edge_index=edge_index_selected, y=label_selected, train_mask=train_mask_selected, val_mask=val_mask_selected, test_mask=test_mask_selected) print(f'num_seleced: {num_selected}/{num}') return graph_pyg_selected, edge_index_unselected def nodeSample(graph_pyg, nodes_per_subgraph): color_print(f'!!!!! Start node sampling') num = graph_pyg.x.shape[0] num_selected = num - num % nodes_per_subgraph subgraphs=[] for i in tqdm(range(num_selected//nodes_per_subgraph)): start_idx = i * nodes_per_subgraph subset=torch.arange(start_idx,start_idx+nodes_per_subgraph) sub_edge_index = torch_geometric.utils.subgraph(subset,graph_pyg.edge_index,num_nodes=graph_pyg.num_nodes)[0] - i*nodes_per_subgraph sub_x=graph_pyg.x[start_idx:start_idx + nodes_per_subgraph] sub_new_x=graph_pyg.new_x[start_idx:start_idx + nodes_per_subgraph] sub_y=graph_pyg.y[start_idx:start_idx + nodes_per_subgraph] sub_adj=to_dense_adj(sub_edge_index,max_num_nodes=nodes_per_subgraph)[0] sub_train_mask=graph_pyg.train_mask[start_idx:start_idx+nodes_per_subgraph] sub_val_mask=graph_pyg.val_mask[start_idx:start_idx+nodes_per_subgraph] sub_test_mask=graph_pyg.test_mask[start_idx:start_idx+nodes_per_subgraph] sub_data=Data(x=sub_x,new_x=sub_new_x,edge_index=sub_edge_index,y=sub_y,adj=sub_adj,train_mask=sub_train_mask,val_mask=sub_val_mask,test_mask=sub_test_mask) subgraphs.append(sub_data) color_print(f'!!!!! Finish node sampling') return subgraphs def sampleCheck(node_groups, nodes_per_subgraph): cnt=0 wrong_num=0 for g in node_groups: if g.x.shape[0]!=nodes_per_subgraph: color_print(g.x.shape[0],cnt) color_print(f'!!!!! Sampling check false') wrong_num=wrong_num+1 cnt=cnt+1 if not wrong_num: color_print(f'!!!!! Sampling check true') def loadDataset(dataset, train_ratio): if dataset=='yelp': graph_dgl = FraudYelpDataset()[0] graph_dgl = dgl.to_homogeneous(graph_dgl,ndata=['feature', 'label', 'train_mask', 'val_mask', 'test_mask']) graph_dgl = dgl.add_self_loop(graph_dgl) feat = graph_dgl.ndata['feature'].float() label = graph_dgl.ndata['label'].long() edge_index = torch.stack(graph_dgl.edges(etype=('_N', '_E', '_N'))) index = list(range(len(label))) train_mask, val_mask, test_mask = splitDataset(index,label,train_ratio) graph_pyg=Data(x=feat,edge_index=edge_index,y=label,train_mask=train_mask,val_mask=val_mask,test_mask=test_mask) graph_dgl.ndata['train_mask']=train_mask graph_dgl.ndata['val_mask']=val_mask graph_dgl.ndata['test_mask']=test_mask elif dataset=='amazon': graph_dgl = FraudAmazonDataset()[0] graph_dgl = dgl.to_homogeneous(graph_dgl,ndata=['feature', 'label', 'train_mask', 'val_mask', 'test_mask']) graph_dgl = dgl.add_self_loop(graph_dgl) feat = graph_dgl.ndata['feature'].float() label = graph_dgl.ndata['label'].long() edge_index = torch.stack(graph_dgl.edges(etype=('_N', '_E', '_N'))) index = list(range(3305, len(label))) train_mask, val_mask, test_mask = splitDataset(index,label,train_ratio) graph_pyg=Data(x=feat,edge_index=edge_index,y=label,train_mask=train_mask,val_mask=val_mask,test_mask=test_mask) graph_dgl.ndata['train_mask']=train_mask graph_dgl.ndata['val_mask']=val_mask graph_dgl.ndata['test_mask']=test_mask elif dataset=='blogcatalog': bc_path = f'../bc.pt' graph_pyg = torch.load(bc_path)[0] graph_pyg.edge_index = graph_pyg.edge_indexes[0] graph_dgl = pyg_data_to_dgl_graph(graph_pyg) label = graph_dgl.ndata['label'].long() index = list(range(len(label))) train_mask, val_mask, test_mask = splitDataset(index,label,train_ratio) graph_pyg.train_mask=train_mask graph_pyg.val_mask=val_mask graph_pyg.test_mask=test_mask graph_dgl.ndata['train_mask']=train_mask graph_dgl.ndata['val_mask']=val_mask graph_dgl.ndata['test_mask']=test_mask else: print(f'{dataset}-graph_dgl not exist') return None color_print(f'{dataset}_graph_dgl: ') print(f'{graph_dgl}') color_print(f'{dataset}_graph_pyg: ') print(f'{graph_pyg}') color_print(f'train_val_test split') print(f'train_num: {train_mask.sum()}; val_num: {val_mask.sum()}; test_num: {test_mask.sum()}') return graph_dgl, graph_pyg, train_mask, val_mask, test_mask def mergeGraphDataList(args, graph_pyg, syn_relation_dict): # 初始化合并后的组件 x, y, new_x, edge_index = [], [], [], [] adj_matrices = [] ddpm_train_mask, ddpm_val_mask, ddpm_test_mask = [], [], [] cum_num_nodes = 0 # 累积节点数,用于更新edge_index new_x_ssupgcl=syn_relation_dict['graph_pyg_ssupgcl_new_x'] for data in syn_relation_dict['syn_relation_list']: # for data in syn_relation_dict['new_data_list']: # 更新节点特征和标签 x.append(data.x) new_x.append(data.new_x) y.append(data.y) # 更新edge_index current_edge_index = data.edge_index + cum_num_nodes edge_index.append(current_edge_index) # 更新邻接矩阵 adj_matrices.append(data.adj) # 更新掩码 ddpm_train_mask.append(data.train_mask) ddpm_val_mask.append(data.val_mask) ddpm_test_mask.append(data.test_mask) # 更新节点累计数 cum_num_nodes += data.num_nodes node_per_subgraph=args.nodes_per_subgraph num_unselected=graph_pyg.x.shape[0] % node_per_subgraph num_unselected # 合并所有组件 new_x = torch.cat(new_x, dim=0) new_x = torch.cat([new_x, new_x_ssupgcl[-num_unselected:]],dim=0) x = torch.cat(x, dim=0) x = torch.cat([x, graph_pyg.x[-num_unselected:]],dim=0) y = torch.cat(y, dim=0) y = torch.cat([y, graph_pyg.y[-num_unselected:]],dim=0) edge_index = torch.cat(edge_index, dim=1) # edge_index = torch.cat([edge_index, ddpm_yelp_unselected_edge_index], dim=1) edge_index = torch.cat([edge_index, syn_relation_dict['unselected_edge_index']], dim=1) # edge_index = torch.cat([edge_index, syn_relation_dict['yelp_unselected_edge_index']], dim=1) ddpm_train_mask = torch.cat(ddpm_train_mask, dim=0) ddpm_train_mask = torch.cat([ddpm_train_mask, graph_pyg.train_mask[-num_unselected:]], dim=0) ddpm_val_mask = torch.cat(ddpm_val_mask, dim=0) ddpm_val_mask = torch.cat([ddpm_val_mask, graph_pyg.val_mask[-num_unselected:]], dim=0) ddpm_test_mask = torch.cat(ddpm_test_mask, dim=0) ddpm_test_mask = torch.cat([ddpm_test_mask, graph_pyg.test_mask[-num_unselected:]], dim=0) # 创建新的Data对象 if args.dataset=='blogcatalog' and args.GuiDDPM_sample_with_guidance: # blogcatalog 要降维 merged_data = Data(x=new_x, edge_index=edge_index, y=y, train_mask=ddpm_train_mask, val_mask=ddpm_val_mask, test_mask=ddpm_test_mask) else: merged_data = Data(x=x, edge_index=edge_index, y=y, train_mask=ddpm_train_mask, val_mask=ddpm_val_mask, test_mask=ddpm_test_mask) return merged_data def GDCAugment(graph_pyg_type, avg_degree): # 应用 GDC 转换 gdc = GDC(self_loop_weight=1, normalization_in='sym', normalization_out='col', diffusion_kwargs=dict(method='ppr', alpha=0.05), sparsification_kwargs=dict(method='threshold', avg_degree=avg_degree), exact=True) graph_pyg_type.transform = gdc gdc_transformed_data = gdc(graph_pyg_type) # 应用 GDC 转换 return gdc_transformed_data def data4WFusionTrain(graph_pyg, graph_syn, graph_gdc_list): new_edges_dict = {} new_edges_dict[('node', 'relation0', 'node')] = (graph_pyg.edge_index[0], graph_pyg.edge_index[1]) new_edges_dict[('node', 'relation1', 'node')] = (graph_syn.edge_index[0], graph_syn.edge_index[1]) for idx, graph_gdc in enumerate(graph_gdc_list, start=2): new_edges_dict[(f'node', f'relation{idx}', f'node')] = (graph_gdc.edge_index[0], graph_gdc.edge_index[1]) # 设置节点总数 num_nodes = graph_syn.x.shape[0] # 创建异构图时显式指定节点的数量 graph_WFusion = dgl.heterograph(new_edges_dict, num_nodes_dict={'node': num_nodes}) # 设置节点特征和标签 graph_WFusion.nodes['node'].data['feature'] = graph_syn.x graph_WFusion.nodes['node'].data['label'] = graph_syn.y graph_WFusion.nodes['node'].data['train_mask'] = graph_syn.train_mask graph_WFusion.nodes['node'].data['test_mask'] = graph_syn.test_mask graph_WFusion.nodes['node'].data['val_mask'] = graph_syn.val_mask return graph_WFusion