Buckets:
| """ | |
| Created on Mar 1, 2020 | |
| Pytorch Implementation of LightGCN in | |
| Xiangnan He et al. LightGCN: Simplifying and Powering Graph Convolution Network for Recommendation | |
| @author: Jianbai Ye (gusye@mail.ustc.edu.cn) | |
| Define models here | |
| """ | |
| import world | |
| import torch | |
| from dataloader import BasicDataset | |
| from torch import nn | |
| import numpy as np | |
| from sentence_transformers import SentenceTransformer | |
| from tqdm import tqdm | |
| class BasicModel(nn.Module): | |
| def __init__(self): | |
| super(BasicModel, self).__init__() | |
| def getUsersRating(self, users): | |
| raise NotImplementedError | |
| class PairWiseModel(BasicModel): | |
| def __init__(self): | |
| super(PairWiseModel, self).__init__() | |
| def bpr_loss(self, users, pos, neg): | |
| """w | |
| Parameters: | |
| users: users list | |
| pos: positive items for corresponding users | |
| neg: negative items for corresponding users | |
| Return: | |
| (log-loss, l2-loss) | |
| """ | |
| raise NotImplementedError | |
| class MLP(nn.Module): | |
| def __init__(self, input_dim=768, output_dim=64): | |
| super(MLP, self).__init__() | |
| self.fc1 = nn.Linear(input_dim, 256) | |
| self.relu = nn.ReLU() | |
| self.leakyrelu = nn.LeakyReLU(0.1, inplace=True) | |
| self.Sigmoid = nn.Sigmoid() | |
| self.fc2 = nn.Linear(256, 128) | |
| self.fc3 = nn.Linear(128, output_dim) | |
| self.fc = nn.Linear(input_dim, output_dim) | |
| def forward(self, x): | |
| torch.autograd.set_detect_anomaly(True) | |
| x = self.fc1(x) | |
| x = self.relu(x) | |
| x = self.fc2(x) | |
| x = self.relu(x) | |
| x = self.fc3(x) | |
| return x | |
| class LightGCN(BasicModel): | |
| def __init__(self, | |
| config:dict, | |
| dataset:BasicDataset): | |
| super(LightGCN, self).__init__() | |
| self.config = config | |
| self.dataset : dataloader.BasicDataset = dataset | |
| self.bert = SentenceTransformer('../../nq-distilbert-base-v1').to(world.device) | |
| self.mlp = MLP(768, self.config['latent_dim_rec']).to(world.device) | |
| self.__init_weight() | |
| def __init_weight(self): | |
| self.num_users = self.dataset.n_users | |
| self.num_items = self.dataset.m_items | |
| self.latent_dim = self.config['latent_dim_rec'] | |
| self.n_layers = self.config['lightGCN_n_layers'] | |
| self.keep_prob = self.config['keep_prob'] | |
| self.A_split = self.config['A_split'] | |
| # self.embedding_user = torch.nn.Embedding( | |
| # num_embeddings=self.num_users, embedding_dim=self.latent_dim).to(world.device) | |
| # self.embedding_item = torch.tensor(self.bert.encode(self.dataset.item_meta)).to(world.device) | |
| # self.embedding_item = torch.nn.Embedding( | |
| # num_embeddings=self.num_items, embedding_dim=self.latent_dim) | |
| if self.config['pretrain'] == 0: | |
| # nn.init.xavier_uniform_(self.embedding_user.weight, gain=1) | |
| # nn.init.xavier_uniform_(self.embedding_item.weight, gain=1) | |
| # print('use xavier initilizer') | |
| # random normal init seems to be a better choice when lightGCN actually don't use any non-linear activation function | |
| # nn.init.normal_(self.embedding_user.weight, std=0.1) | |
| # self.embedding_item = torch.nn.functional.normalize(self.embedding_item, p = 2, dim = -1) | |
| # nn.init.normal_(self.embedding_item, std=0.1) | |
| world.cprint('use NORMAL distribution initilizer') | |
| else: | |
| self.embedding_user.weight.data.copy_(torch.from_numpy(self.config['user_emb'])) | |
| self.embedding_item.weight.data.copy_(torch.from_numpy(self.config['item_emb'])) | |
| print('use pretarined data') | |
| self.f = nn.Sigmoid() | |
| self.Graph = self.dataset.getSparseGraph() | |
| print(f"lgn is already to go(dropout:{self.config['dropout']})") | |
| # print("save_txt") | |
| def __dropout_x(self, x, keep_prob): | |
| size = x.size() | |
| index = x.indices().t() | |
| values = x.values() | |
| random_index = torch.rand(len(values)) + keep_prob | |
| random_index = random_index.int().bool() | |
| index = index[random_index] | |
| values = values[random_index]/keep_prob | |
| g = torch.sparse.FloatTensor(index.t(), values, size) | |
| return g | |
| def __dropout(self, keep_prob): | |
| if self.A_split: | |
| graph = [] | |
| for g in self.Graph: | |
| graph.append(self.__dropout_x(g, keep_prob)) | |
| else: | |
| graph = self.__dropout_x(self.Graph, keep_prob) | |
| return graph | |
| def computer(self): | |
| """ | |
| propagate methods for lightGCN | |
| """ | |
| word_embeddings = self.bert.encode(self.dataset.item_meta, convert_to_numpy=True, convert_to_tensor=False, normalize_embeddings=True) | |
| self.item_word_embeddings = torch.tensor(word_embeddings).to(world.device) | |
| self.embedding_item = self.mlp(self.item_word_embeddings).to(world.device) | |
| review_embeddings = self.bert.encode(self.dataset.userReviews, convert_to_numpy=True) | |
| user_embeddings = [] | |
| currIndex = 0 | |
| for i in self.dataset.trainUserCount: | |
| print(currIndex, i) | |
| mean_emb = np.mean(review_embeddings[currIndex:currIndex+i], axis=0) | |
| # users_emb = nn.functional.normalize(torch.tensor(mean_emb), p = 2, dim = -1) | |
| currIndex += i | |
| user_embeddings.append(mean_emb) | |
| user_embeddings = np.array(user_embeddings) | |
| user_embeddings = torch.tensor(user_embeddings) | |
| user_embeddings = nn.functional.normalize(user_embeddings).to(world.device) | |
| # user_embeddings = torch.stack(user_embeddings, dim = 0).to(world.device) | |
| self.embedding_user = self.mlp(user_embeddings).to(world.device) | |
| # self.embedding_item = torch.nn.functional.normalize(self.embedding_item, p = 2, dim = -1) | |
| users_emb = self.embedding_user | |
| items_emb = self.embedding_item | |
| all_emb = torch.cat([users_emb, items_emb]) | |
| embs = [all_emb] | |
| if self.config['dropout']: | |
| if self.training: | |
| print("droping") | |
| g_droped = self.__dropout(self.keep_prob) | |
| else: | |
| g_droped = self.Graph | |
| else: | |
| g_droped = self.Graph | |
| for layer in range(self.n_layers): | |
| if self.A_split: | |
| temp_emb = [] | |
| for f in range(len(g_droped)): | |
| temp_emb.append(torch.sparse.mm(g_droped[f], all_emb)) | |
| side_emb = torch.cat(temp_emb, dim=0) | |
| all_emb = side_emb | |
| else: | |
| all_emb = torch.sparse.mm(g_droped.to(world.device), all_emb.to(world.device)) | |
| embs.append(all_emb) | |
| embs = torch.stack(embs, dim=1) | |
| light_out = torch.mean(embs, dim=1) | |
| users, items = torch.split(light_out, [self.num_users, self.num_items]) | |
| return users, items | |
| def getUsersRating(self, users): | |
| all_users, all_items = self.computer() | |
| users_emb = all_users[users.long()] | |
| items_emb = all_items | |
| rating = self.f(torch.matmul(users_emb, items_emb.t())) | |
| return rating | |
| def getEmbedding(self, users, pos_items, neg_items): | |
| all_users, all_items = self.computer() | |
| users_emb = all_users[users] | |
| pos_emb = all_items[pos_items] | |
| neg_emb = all_items[neg_items] | |
| users_emb_ego = self.embedding_user[users] | |
| pos_emb_ego = self.embedding_item[pos_items] | |
| neg_emb_ego = self.embedding_item[neg_items] | |
| return users_emb, pos_emb, neg_emb, users_emb_ego, pos_emb_ego, neg_emb_ego | |
| def bpr_loss(self, users, pos, neg): | |
| (users_emb, pos_emb, neg_emb, | |
| userEmb0, posEmb0, negEmb0) = self.getEmbedding(users.long(), pos.long(), neg.long()) | |
| reg_loss = (1/2)*(userEmb0.norm(2).pow(2) + | |
| posEmb0.norm(2).pow(2) + | |
| negEmb0.norm(2).pow(2))/float(len(users)) | |
| pos_scores = torch.mul(users_emb, pos_emb) | |
| pos_scores = torch.sum(pos_scores, dim=1) | |
| neg_scores = torch.mul(users_emb, neg_emb) | |
| neg_scores = torch.sum(neg_scores, dim=1) | |
| loss = torch.mean(torch.nn.functional.softplus(neg_scores - pos_scores)) | |
| return loss, reg_loss | |
| def forward(self, users, items): | |
| # compute embedding | |
| all_users, all_items = self.computer() | |
| users_emb = all_users[users] | |
| items_emb = all_items[items] | |
| inner_pro = torch.mul(users_emb, items_emb) | |
| gamma = torch.sum(inner_pro, dim=1) | |
| return gamma | |
Xet Storage Details
- Size:
- 8.66 kB
- Xet hash:
- c353039f4c36c1c4dcbb1128e449f683c52990b6f327c6bca1fc0e84f0613b2d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.