Transformers
PyTorch
FolashadeO's picture
Upload model.py with huggingface_hub
d31659e verified
Raw History Blame Contribute Delete
1.49 kB
import torch
import torch.nn as nn
from transformers import BertModel
from transformers.modeling_outputs import TokenClassifierOutput
class BertCrfForTokenClassification(nn.Module):
def __init__(self, model_name_or_path: str, num_labels: int):
super().__init__()
self.num_labels = num_labels
self.bert = BertModel.from_pretrained(model_name_or_path, add_pooling_layer=False)
self.dropout = nn.Dropout(0.1)
self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels)
def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
sequence_output = self.dropout(outputs[0])
emissions = self.classifier(sequence_output)
loss = None
if labels is not None:
loss_fct = nn.CrossEntropyLoss()
if attention_mask is not None:
active_loss = attention_mask.view(-1) == 1
active_logits = emissions.view(-1, self.num_labels)
active_labels = torch.where(
active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index).type_as(labels)
)
loss = loss_fct(active_logits, active_labels)
else:
loss = loss_fct(emissions.view(-1, self.num_labels), labels.view(-1))
return TokenClassifierOutput(loss=loss, logits=emissions)