Upload 7 files

3d3bb36 over 3 years ago

3.88 kB

import torch.nn as nn
from transformers import BertModel
from .config import *
import torch
import torch.nn.functional as F

# 忽略 transformers 警告
from transformers import logging
logging.set_verbosity_error()


class CasRel(nn.Module):

    #初始化model
    def __init__(self):
        super().__init__()
        self.bert = BertModel.from_pretrained(BERT_MODEL_NAME)
        # 冻结Bert参数，只训练下游模型
        for name, param in self.bert.named_parameters():
            param.requires_grad = False

        #定义CasRel网络
        self.sub_head_linear = nn.Linear(BERT_DIM, 1)   #sub只需要一维即
        self.sub_tail_linear = nn.Linear(BERT_DIM, 1)

        self.obj_head_linear = nn.Linear(BERT_DIM, REL_SIZE)  #预测的obj矩阵需要REL_SIZE维计
        self.obj_tail_linear = nn.Linear(BERT_DIM, REL_SIZE)


    #subject头尾标记预测
    def get_encoded_text(self, input_ids, mask):
        return self.bert(input_ids, attention_mask=mask)[0]

    def get_subs(self, encoded_text):
        #encoded_text(b,c,768) -> (b,c,1)
        #对每个单词进行sigmoid预测
        pred_sub_head = torch.sigmoid(self.sub_head_linear(encoded_text))
        pred_sub_tail = torch.sigmoid(self.sub_tail_linear(encoded_text))

        return pred_sub_head, pred_sub_tail

    def get_objs_for_specific_sub(self, encoded_text, sub_head_seq, sub_tail_seq):  # 获得预测的obj-rel矩阵
        # sub_head_seq.shape (b, c) -> (b, 1, c)
        sub_head_seq = sub_head_seq.unsqueeze(1).float()
        sub_tail_seq = sub_tail_seq.unsqueeze(1).float()

        # encoded_text.shape (b, c, 768)
        sub_head = torch.matmul(sub_head_seq, encoded_text)   #获得head和tail的编码并加在encoded_text中
        sub_tail = torch.matmul(sub_tail_seq, encoded_text)

        encoded_text = encoded_text + (sub_head + sub_tail) / 2

        # encoded_text.shape (b, c, 768)
        pred_obj_head = torch.sigmoid(self.obj_head_linear(encoded_text))
        pred_obj_tail = torch.sigmoid(self.obj_tail_linear(encoded_text))

        # shape (b, c, REL_SIZE)
        return pred_obj_head, pred_obj_tail

    def forward(self, input, mask):

        input_ids, sub_head_seq, sub_tail_seq = input
        encoded_text = self.get_encoded_text(input_ids, mask)
        pred_sub_head, pred_sub_tail = self.get_subs(encoded_text)


        input_ids, sub_head_seq, sub_tail_seq = input
        encoded_text = self.get_encoded_text(input_ids, mask)

        # 预测subject首尾序列
        pred_sub_head, pred_sub_tail = self.get_subs(encoded_text)

        # 预测relation-object矩阵
        pred_obj_head, pred_obj_tail = self.get_objs_for_specific_sub(encoded_text, sub_head_seq, sub_tail_seq)

        return encoded_text, (pred_sub_head, pred_sub_tail, pred_obj_head, pred_obj_tail)

    def loss_fn(self, true_y, pred_y, mask):

        def calc_loss(pred, true, mask):
            true = true.float()

            # pred.shape (b, c, 1) -> (b, c)
            pred = pred.squeeze(-1)
            weight = torch.where(true > 0, CLS_WEIGHT_COEF[1], CLS_WEIGHT_COEF[0])  # 分配权重


            loss = F.binary_cross_entropy(pred, true, weight=weight, reduction='none')


            if loss.shape != mask.shape:
                mask = mask.unsqueeze(-1)

            return torch.sum(loss * mask) / torch.sum(mask)  #通过与mask相乘将pad补充的元素损失进行归0

        pred_sub_head, pred_sub_tail, pred_obj_head, pred_obj_tail = pred_y
        true_sub_head, true_sub_tail, true_obj_head, true_obj_tail = true_y


        return calc_loss(pred_sub_head, true_sub_head, mask) * SUB_WEIGHT_COEF + \
               calc_loss(pred_sub_tail, true_sub_tail, mask) * SUB_WEIGHT_COEF + \
               calc_loss(pred_obj_head, true_obj_head, mask) + \
               calc_loss(pred_obj_tail, true_obj_tail, mask)

	import torch.nn as nn
	from transformers import BertModel
	from .config import *
	import torch
	import torch.nn.functional as F

	# 忽略 transformers 警告
	from transformers import logging
	logging.set_verbosity_error()


	class CasRel(nn.Module):

	#初始化model
	def __init__(self):
	super().__init__()
	self.bert = BertModel.from_pretrained(BERT_MODEL_NAME)
	# 冻结Bert参数，只训练下游模型
	for name, param in self.bert.named_parameters():
	param.requires_grad = False

	#定义CasRel网络
	self.sub_head_linear = nn.Linear(BERT_DIM, 1) #sub只需要一维即
	self.sub_tail_linear = nn.Linear(BERT_DIM, 1)

	self.obj_head_linear = nn.Linear(BERT_DIM, REL_SIZE) #预测的obj矩阵需要REL_SIZE维计
	self.obj_tail_linear = nn.Linear(BERT_DIM, REL_SIZE)


	#subject头尾标记预测
	def get_encoded_text(self, input_ids, mask):
	return self.bert(input_ids, attention_mask=mask)[0]

	def get_subs(self, encoded_text):
	#encoded_text(b,c,768) -> (b,c,1)
	#对每个单词进行sigmoid预测
	pred_sub_head = torch.sigmoid(self.sub_head_linear(encoded_text))
	pred_sub_tail = torch.sigmoid(self.sub_tail_linear(encoded_text))

	return pred_sub_head, pred_sub_tail

	def get_objs_for_specific_sub(self, encoded_text, sub_head_seq, sub_tail_seq): # 获得预测的obj-rel矩阵
	# sub_head_seq.shape (b, c) -> (b, 1, c)
	sub_head_seq = sub_head_seq.unsqueeze(1).float()
	sub_tail_seq = sub_tail_seq.unsqueeze(1).float()

	# encoded_text.shape (b, c, 768)
	sub_head = torch.matmul(sub_head_seq, encoded_text) #获得head和tail的编码并加在encoded_text中
	sub_tail = torch.matmul(sub_tail_seq, encoded_text)

	encoded_text = encoded_text + (sub_head + sub_tail) / 2

	# encoded_text.shape (b, c, 768)
	pred_obj_head = torch.sigmoid(self.obj_head_linear(encoded_text))
	pred_obj_tail = torch.sigmoid(self.obj_tail_linear(encoded_text))

	# shape (b, c, REL_SIZE)
	return pred_obj_head, pred_obj_tail

	def forward(self, input, mask):

	input_ids, sub_head_seq, sub_tail_seq = input
	encoded_text = self.get_encoded_text(input_ids, mask)
	pred_sub_head, pred_sub_tail = self.get_subs(encoded_text)


	input_ids, sub_head_seq, sub_tail_seq = input
	encoded_text = self.get_encoded_text(input_ids, mask)

	# 预测subject首尾序列
	pred_sub_head, pred_sub_tail = self.get_subs(encoded_text)

	# 预测relation-object矩阵
	pred_obj_head, pred_obj_tail = self.get_objs_for_specific_sub(encoded_text, sub_head_seq, sub_tail_seq)

	return encoded_text, (pred_sub_head, pred_sub_tail, pred_obj_head, pred_obj_tail)

	def loss_fn(self, true_y, pred_y, mask):

	def calc_loss(pred, true, mask):
	true = true.float()

	# pred.shape (b, c, 1) -> (b, c)
	pred = pred.squeeze(-1)
	weight = torch.where(true > 0, CLS_WEIGHT_COEF[1], CLS_WEIGHT_COEF[0]) # 分配权重


	loss = F.binary_cross_entropy(pred, true, weight=weight, reduction='none')


	if loss.shape != mask.shape:
	mask = mask.unsqueeze(-1)

	return torch.sum(loss * mask) / torch.sum(mask) #通过与mask相乘将pad补充的元素损失进行归0

	pred_sub_head, pred_sub_tail, pred_obj_head, pred_obj_tail = pred_y
	true_sub_head, true_sub_tail, true_obj_head, true_obj_tail = true_y


	return calc_loss(pred_sub_head, true_sub_head, mask) * SUB_WEIGHT_COEF + \
	calc_loss(pred_sub_tail, true_sub_tail, mask) * SUB_WEIGHT_COEF + \
	calc_loss(pred_obj_head, true_obj_head, mask) + \
	calc_loss(pred_obj_tail, true_obj_tail, mask)