Buckets:
| """ | |
| Created on Mar 1, 2020 | |
| Pytorch Implementation of LightGCN in | |
| Xiangnan He et al. LightGCN: Simplifying and Powering Graph Convolution Network for Recommendation | |
| @author: Shuxian Bi (stanbi@mail.ustc.edu.cn),Jianbai Ye (gusye@mail.ustc.edu.cn) | |
| Design Dataset here | |
| Every dataset's index has to start at 0 | |
| """ | |
| import os | |
| from os.path import join | |
| import sys | |
| import torch | |
| import numpy as np | |
| import pandas as pd | |
| from torch.utils.data import Dataset, DataLoader | |
| from scipy.sparse import csr_matrix | |
| import scipy.sparse as sp | |
| import world | |
| from world import cprint | |
| from time import time | |
| class BasicDataset(Dataset): | |
| def __init__(self): | |
| print("init dataset") | |
| def n_users(self): | |
| raise NotImplementedError | |
| def m_items(self): | |
| raise NotImplementedError | |
| def trainDataSize(self): | |
| raise NotImplementedError | |
| def testDict(self): | |
| raise NotImplementedError | |
| def allPos(self): | |
| raise NotImplementedError | |
| def getUserItemFeedback(self, users, items): | |
| raise NotImplementedError | |
| def getUserPosItems(self, users): | |
| raise NotImplementedError | |
| def getUserNegItems(self, users): | |
| """ | |
| not necessary for large dataset | |
| it's stupid to return all neg items in super large dataset | |
| """ | |
| raise NotImplementedError | |
| def getSparseGraph(self): | |
| """ | |
| build a graph in torch.sparse.IntTensor. | |
| Details in NGCF's matrix form | |
| A = | |
| |I, R| | |
| |R^T, I| | |
| """ | |
| raise NotImplementedError | |
| class LastFM(BasicDataset): | |
| """ | |
| Dataset type for pytorch \n | |
| Incldue graph information | |
| LastFM dataset | |
| """ | |
| def __init__(self, path="./lastfm"): | |
| # train or test | |
| cprint("loading [last fm]") | |
| self.mode_dict = {'train':0, "test":1} | |
| self.mode = self.mode_dict['train'] | |
| # self.n_users = 1892 | |
| # self.m_items = 4489 | |
| trainData = pd.read_table(join(path, 'data1.txt'), header=None) | |
| # print(trainData.head()) | |
| testData = pd.read_table(join(path, 'test1.txt'), header=None) | |
| # print(testData.head()) | |
| trustNet = pd.read_table(join(path, 'trustnetwork.txt'), header=None).to_numpy() | |
| # print(trustNet[:5]) | |
| trustNet -= 1 | |
| trainData-= 1 | |
| testData -= 1 | |
| self.trustNet = trustNet | |
| self.trainData = trainData | |
| self.testData = testData | |
| self.trainUser = np.array(trainData[:][0]) | |
| self.trainUniqueUsers = np.unique(self.trainUser) | |
| self.trainItem = np.array(trainData[:][1]) | |
| # self.trainDataSize = len(self.trainUser) | |
| self.testUser = np.array(testData[:][0]) | |
| self.testUniqueUsers = np.unique(self.testUser) | |
| self.testItem = np.array(testData[:][1]) | |
| self.Graph = None | |
| print(f"LastFm Sparsity : {(len(self.trainUser) + len(self.testUser))/self.n_users/self.m_items}") | |
| # (users,users) | |
| self.socialNet = csr_matrix((np.ones(len(trustNet)), (trustNet[:,0], trustNet[:,1]) ), shape=(self.n_users,self.n_users)) | |
| # (users,items), bipartite graph | |
| self.UserItemNet = csr_matrix((np.ones(len(self.trainUser)), (self.trainUser, self.trainItem) ), shape=(self.n_users,self.m_items)) | |
| # pre-calculate | |
| self._allPos = self.getUserPosItems(list(range(self.n_users))) | |
| self.allNeg = [] | |
| allItems = set(range(self.m_items)) | |
| for i in range(self.n_users): | |
| pos = set(self._allPos[i]) | |
| neg = allItems - pos | |
| self.allNeg.append(np.array(list(neg))) | |
| self.__testDict = self.__build_test() | |
| def n_users(self): | |
| return 1892 | |
| def m_items(self): | |
| return 4489 | |
| def trainDataSize(self): | |
| return len(self.trainUser) | |
| def testDict(self): | |
| return self.__testDict | |
| def allPos(self): | |
| return self._allPos | |
| def getSparseGraph(self): | |
| if self.Graph is None: | |
| user_dim = torch.LongTensor(self.trainUser) | |
| item_dim = torch.LongTensor(self.trainItem) | |
| first_sub = torch.stack([user_dim, item_dim + self.n_users]) | |
| second_sub = torch.stack([item_dim+self.n_users, user_dim]) | |
| index = torch.cat([first_sub, second_sub], dim=1) | |
| data = torch.ones(index.size(-1)).int() | |
| self.Graph = torch.sparse.IntTensor(index, data, torch.Size([self.n_users+self.m_items, self.n_users+self.m_items])) | |
| dense = self.Graph.to_dense() | |
| D = torch.sum(dense, dim=1).float() | |
| D[D==0.] = 1. | |
| D_sqrt = torch.sqrt(D).unsqueeze(dim=0) | |
| dense = dense/D_sqrt | |
| dense = dense/D_sqrt.t() | |
| index = dense.nonzero() | |
| data = dense[dense >= 1e-9] | |
| assert len(index) == len(data) | |
| self.Graph = torch.sparse.FloatTensor(index.t(), data, torch.Size([self.n_users+self.m_items, self.n_users+self.m_items])) | |
| self.Graph = self.Graph.coalesce().to(world.device) | |
| return self.Graph | |
| def __build_test(self): | |
| """ | |
| return: | |
| dict: {user: [items]} | |
| """ | |
| test_data = {} | |
| for i, item in enumerate(self.testItem): | |
| user = self.testUser[i] | |
| if test_data.get(user): | |
| test_data[user].append(item) | |
| else: | |
| test_data[user] = [item] | |
| return test_data | |
| def getUserItemFeedback(self, users, items): | |
| """ | |
| users: | |
| shape [-1] | |
| items: | |
| shape [-1] | |
| return: | |
| feedback [-1] | |
| """ | |
| # print(self.UserItemNet[users, items]) | |
| return np.array(self.UserItemNet[users, items]).astype('uint8').reshape((-1, )) | |
| def getUserPosItems(self, users): | |
| posItems = [] | |
| for user in users: | |
| posItems.append(self.UserItemNet[user].nonzero()[1]) | |
| return posItems | |
| def getUserNegItems(self, users): | |
| negItems = [] | |
| for user in users: | |
| negItems.append(self.allNeg[user]) | |
| return negItems | |
| def __getitem__(self, index): | |
| user = self.trainUniqueUsers[index] | |
| # return user_id and the positive items of the user | |
| return user | |
| def switch2test(self): | |
| """ | |
| change dataset mode to offer test data to dataloader | |
| """ | |
| self.mode = self.mode_dict['test'] | |
| def __len__(self): | |
| return len(self.trainUniqueUsers) | |
| class Loader(BasicDataset): | |
| """ | |
| Dataset type for pytorch \n | |
| Incldue graph information | |
| gowalla dataset | |
| """ | |
| def __init__(self,config = world.config,path="./data/Movie_and_TV"): | |
| # train or test | |
| cprint(f'loading [{path}]') | |
| self.split = config['A_split'] | |
| self.folds = config['A_n_fold'] | |
| self.mode_dict = {'train': 0, "test": 1} | |
| self.mode = self.mode_dict['train'] | |
| self.n_user = 0 | |
| self.m_item = 0 | |
| train_file = path + '/train.txt' | |
| test_file = path + '/test.txt' | |
| item_dict_file = path + '/item_dict.txt' | |
| review_file = path + '/review_dict.txt' | |
| self.item_meta = [] | |
| self.path = path | |
| trainUniqueUsers, trainItem, trainUser = [], [], [] | |
| testUniqueUsers, testItem, testUser = [], [], [] | |
| self.trainUserCount = [] | |
| self.traindataSize = 0 | |
| self.testDataSize = 0 | |
| self.userReviews = [] | |
| self.interByUser = [] | |
| with open(train_file) as f: | |
| for l in f.readlines(): | |
| if len(l) > 0: | |
| l = l.strip('\n').split(' ') | |
| items = [int(i) for i in l[1:]] | |
| uid = int(l[0]) | |
| self.interByUser.append(items) | |
| trainUniqueUsers.append(uid) | |
| trainUser.extend([uid] * len(items)) | |
| trainItem.extend(items) | |
| self.m_item = max(self.m_item, max(items)) | |
| self.n_user = max(self.n_user, uid) | |
| self.traindataSize += len(items) | |
| self.trainUniqueUsers = np.array(trainUniqueUsers) | |
| self.trainUser = np.array(trainUser) | |
| self.trainItem = np.array(trainItem) | |
| with open(test_file) as f: | |
| for l in f.readlines(): | |
| if len(l) > 0: | |
| l = l.strip('\n').split(' ') | |
| items = [int(i) for i in l[1:]] | |
| uid = int(l[0]) | |
| testUniqueUsers.append(uid) | |
| testUser.extend([uid] * len(items)) | |
| testItem.extend(items) | |
| self.m_item = max(self.m_item, max(items)) | |
| self.n_user = max(self.n_user, uid) | |
| self.testDataSize += len(items) | |
| self.m_item += 1 | |
| self.n_user += 1 | |
| self.testUniqueUsers = np.array(testUniqueUsers) | |
| self.testUser = np.array(testUser) | |
| self.testItem = np.array(testItem) | |
| item_with_meta = 0 | |
| with open(item_dict_file) as f: | |
| for l in f.readlines(): | |
| if len(l) > 0: | |
| l = l.strip('\n').split(' ', 1) | |
| self.item_meta.append(l[1]) | |
| item_with_meta += 1 | |
| print(item_with_meta) | |
| print(self.m_item) | |
| assert item_with_meta == self.m_item | |
| currUser = 0 | |
| currCount = 0 | |
| with open(review_file) as f: | |
| for l in f.readlines(): | |
| if len(l) > 0: | |
| l = l.strip('\n').split(' ', 2) | |
| uid = int(l[0]) | |
| if uid != currUser: | |
| self.trainUserCount.append(currCount) | |
| currCount = 0 | |
| currUser = uid | |
| currCount += 1 | |
| self.userReviews.append(l[2]) | |
| self.trainUserCount.append(currCount) | |
| self.Graph = None | |
| print(f"{self.trainDataSize} interactions for training") | |
| print(f"{self.testDataSize} interactions for testing") | |
| print(f"{world.dataset} Sparsity : {(self.trainDataSize + self.testDataSize) / self.n_users / self.m_items}") | |
| # (users,items), bipartite graph | |
| self.UserItemNet = csr_matrix((np.ones(len(self.trainUser)), (self.trainUser, self.trainItem)), | |
| shape=(self.n_user, self.m_item)) | |
| self.users_D = np.array(self.UserItemNet.sum(axis=1)).squeeze() | |
| self.users_D[self.users_D == 0.] = 1 | |
| self.items_D = np.array(self.UserItemNet.sum(axis=0)).squeeze() | |
| self.items_D[self.items_D == 0.] = 1. | |
| # pre-calculate | |
| self._allPos = self.getUserPosItems(list(range(self.n_user))) | |
| self.__testDict = self.__build_test() | |
| print(f"{world.dataset} is ready to go") | |
| def n_users(self): | |
| return self.n_user | |
| def m_items(self): | |
| return self.m_item | |
| def trainDataSize(self): | |
| return self.traindataSize | |
| def testDict(self): | |
| return self.__testDict | |
| def allPos(self): | |
| return self._allPos | |
| def _split_A_hat(self,A): | |
| A_fold = [] | |
| fold_len = (self.n_users + self.m_items) // self.folds | |
| for i_fold in range(self.folds): | |
| start = i_fold*fold_len | |
| if i_fold == self.folds - 1: | |
| end = self.n_users + self.m_items | |
| else: | |
| end = (i_fold + 1) * fold_len | |
| A_fold.append(self._convert_sp_mat_to_sp_tensor(A[start:end]).coalesce().to(world.device)) | |
| return A_fold | |
| def _convert_sp_mat_to_sp_tensor(self, X): | |
| coo = X.tocoo().astype(np.float32) | |
| row = torch.Tensor(coo.row).long() | |
| col = torch.Tensor(coo.col).long() | |
| index = torch.stack([row, col]) | |
| data = torch.FloatTensor(coo.data) | |
| return torch.sparse.FloatTensor(index, data, torch.Size(coo.shape)) | |
| def getSparseGraph(self): | |
| print("loading adjacency matrix") | |
| if self.Graph is None: | |
| try: | |
| pre_adj_mat = sp.load_npz(self.path + '/s_pre_adj_mat.npz') | |
| print("successfully loaded...") | |
| norm_adj = pre_adj_mat | |
| except : | |
| print("generating adjacency matrix") | |
| s = time() | |
| adj_mat = sp.dok_matrix((self.n_users + self.m_items, self.n_users + self.m_items), dtype=np.float32) | |
| adj_mat = adj_mat.tolil() | |
| R = self.UserItemNet.tolil() | |
| adj_mat[:self.n_users, self.n_users:] = R | |
| adj_mat[self.n_users:, :self.n_users] = R.T | |
| adj_mat = adj_mat.todok() | |
| # adj_mat = adj_mat + sp.eye(adj_mat.shape[0]) | |
| rowsum = np.array(adj_mat.sum(axis=1)) | |
| d_inv = np.power(rowsum, -0.5).flatten() | |
| d_inv[np.isinf(d_inv)] = 0. | |
| d_mat = sp.diags(d_inv) | |
| norm_adj = d_mat.dot(adj_mat) | |
| norm_adj = norm_adj.dot(d_mat) | |
| norm_adj = norm_adj.tocsr() | |
| end = time() | |
| print(f"costing {end-s}s, saved norm_mat...") | |
| sp.save_npz(self.path + '/s_pre_adj_mat.npz', norm_adj) | |
| if self.split == True: | |
| self.Graph = self._split_A_hat(norm_adj) | |
| print("done split matrix") | |
| else: | |
| self.Graph = self._convert_sp_mat_to_sp_tensor(norm_adj) | |
| self.Graph = self.Graph.coalesce().to(world.device) | |
| print("don't split the matrix") | |
| return self.Graph | |
| def __build_test(self): | |
| """ | |
| return: | |
| dict: {user: [items]} | |
| """ | |
| test_data = {} | |
| for i, item in enumerate(self.testItem): | |
| user = self.testUser[i] | |
| if test_data.get(user): | |
| test_data[user].append(item) | |
| else: | |
| test_data[user] = [item] | |
| return test_data | |
| def getUserItemFeedback(self, users, items): | |
| """ | |
| users: | |
| shape [-1] | |
| items: | |
| shape [-1] | |
| return: | |
| feedback [-1] | |
| """ | |
| # print(self.UserItemNet[users, items]) | |
| return np.array(self.UserItemNet[users, items]).astype('uint8').reshape((-1,)) | |
| def getUserPosItems(self, users): | |
| posItems = [] | |
| for user in users: | |
| posItems.append(self.UserItemNet[user].nonzero()[1]) | |
| return posItems | |
| # def getUserNegItems(self, users): | |
| # negItems = [] | |
| # for user in users: | |
| # negItems.append(self.allNeg[user]) | |
| # return negItems | |
Xet Storage Details
- Size:
- 15 kB
- Xet hash:
- d527818e60b47968dbf9898940860b4e07114cffccfd53c3732d18d2ada28f51
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.