import torch import torch.nn as nn from transformers import BertModel from transformers.modeling_outputs import TokenClassifierOutput class BertCrfForTokenClassification(nn.Module): def __init__(self, model_name_or_path: str, num_labels: int): super().__init__() self.num_labels = num_labels self.bert = BertModel.from_pretrained(model_name_or_path, add_pooling_layer=False) self.dropout = nn.Dropout(0.1) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None): outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) sequence_output = self.dropout(outputs[0]) emissions = self.classifier(sequence_output) loss = None if labels is not None: loss_fct = nn.CrossEntropyLoss() if attention_mask is not None: active_loss = attention_mask.view(-1) == 1 active_logits = emissions.view(-1, self.num_labels) active_labels = torch.where( active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index).type_as(labels) ) loss = loss_fct(active_logits, active_labels) else: loss = loss_fct(emissions.view(-1, self.num_labels), labels.view(-1)) return TokenClassifierOutput(loss=loss, logits=emissions)