Spaces:
Sleeping
Sleeping
Delete model.py
Browse files
model.py
DELETED
|
@@ -1,140 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
import torch.nn as nn
|
| 3 |
-
import torch.optim as optim
|
| 4 |
-
import numpy as np
|
| 5 |
-
from transformers import BertTokenizer, BertModel
|
| 6 |
-
from datasets import load_dataset
|
| 7 |
-
from sklearn.model_selection import train_test_split
|
| 8 |
-
from torch.utils.data import Dataset, DataLoader
|
| 9 |
-
from tqdm import tqdm
|
| 10 |
-
from sklearn.metrics import accuracy_score, f1_score
|
| 11 |
-
|
| 12 |
-
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 13 |
-
|
| 14 |
-
dataset = load_dataset("go_emotions")
|
| 15 |
-
|
| 16 |
-
# Extract text and labels
|
| 17 |
-
texts = dataset["train"]["text"][:20000] # Increased dataset size
|
| 18 |
-
labels = dataset["train"]["labels"][:20000] # Increased dataset size
|
| 19 |
-
|
| 20 |
-
# Convert labels to categorical
|
| 21 |
-
def fix_labels(labels):
|
| 22 |
-
labels = [max(label) if label else 0 for label in labels] # Convert multi-label to single-label
|
| 23 |
-
return torch.tensor(labels, dtype=torch.long)
|
| 24 |
-
|
| 25 |
-
labels = fix_labels(labels)
|
| 26 |
-
|
| 27 |
-
# Split dataset
|
| 28 |
-
train_texts, val_texts, train_labels, val_labels = train_test_split(texts, labels, test_size=0.2, random_state=42)
|
| 29 |
-
|
| 30 |
-
# Tokenizer
|
| 31 |
-
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
|
| 32 |
-
|
| 33 |
-
# Tokenize text
|
| 34 |
-
def tokenize(texts):
|
| 35 |
-
return tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
|
| 36 |
-
|
| 37 |
-
train_encodings = tokenize(train_texts)
|
| 38 |
-
val_encodings = tokenize(val_texts)
|
| 39 |
-
train_encodings = {key: val.to(device) for key, val in train_encodings.items()}
|
| 40 |
-
val_encodings = {key: val.to(device) for key, val in val_encodings.items()}
|
| 41 |
-
|
| 42 |
-
class EmotionDataset(Dataset):
|
| 43 |
-
def __init__(self, encodings, labels):
|
| 44 |
-
self.encodings = encodings
|
| 45 |
-
self.labels = labels
|
| 46 |
-
|
| 47 |
-
def __len__(self):
|
| 48 |
-
return len(self.labels)
|
| 49 |
-
|
| 50 |
-
def __getitem__(self, idx):
|
| 51 |
-
item = {key: val[idx] for key, val in self.encodings.items()}
|
| 52 |
-
item["labels"] = self.labels[idx]
|
| 53 |
-
return item
|
| 54 |
-
|
| 55 |
-
train_dataset = EmotionDataset(train_encodings, train_labels)
|
| 56 |
-
val_dataset = EmotionDataset(val_encodings, val_labels)
|
| 57 |
-
|
| 58 |
-
train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
|
| 59 |
-
val_loader = DataLoader(val_dataset, batch_size=16)
|
| 60 |
-
|
| 61 |
-
class BertGRUClassifier(nn.Module):
|
| 62 |
-
def __init__(self, bert_model="bert-base-uncased", hidden_dim=128, num_classes=28):
|
| 63 |
-
super(BertGRUClassifier, self).__init__()
|
| 64 |
-
self.bert = BertModel.from_pretrained(bert_model)
|
| 65 |
-
self.gru = nn.GRU(self.bert.config.hidden_size, hidden_dim, batch_first=True)
|
| 66 |
-
self.dropout = nn.Dropout(0.3) # Added dropout layer
|
| 67 |
-
self.fc = nn.Linear(hidden_dim, num_classes)
|
| 68 |
-
|
| 69 |
-
def forward(self, input_ids, attention_mask):
|
| 70 |
-
with torch.no_grad():
|
| 71 |
-
bert_output = self.bert(input_ids=input_ids, attention_mask=attention_mask)
|
| 72 |
-
gru_output, _ = self.gru(bert_output.last_hidden_state)
|
| 73 |
-
output = self.fc(self.dropout(gru_output[:, -1, :])) # Apply dropout
|
| 74 |
-
return output
|
| 75 |
-
|
| 76 |
-
model = BertGRUClassifier()
|
| 77 |
-
model.to(device)
|
| 78 |
-
|
| 79 |
-
criterion = nn.CrossEntropyLoss()
|
| 80 |
-
optimizer = optim.Adam(model.parameters(), lr=2e-5)
|
| 81 |
-
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1) # Added learning rate scheduler
|
| 82 |
-
|
| 83 |
-
def evaluate_model(model, data_loader):
|
| 84 |
-
model.eval()
|
| 85 |
-
predictions, true_labels = [], []
|
| 86 |
-
|
| 87 |
-
with torch.no_grad():
|
| 88 |
-
for batch in data_loader:
|
| 89 |
-
input_ids = batch["input_ids"].to(device)
|
| 90 |
-
attention_mask = batch["attention_mask"].to(device)
|
| 91 |
-
labels = batch["labels"].to(device)
|
| 92 |
-
|
| 93 |
-
outputs = model(input_ids, attention_mask)
|
| 94 |
-
preds = torch.argmax(outputs, dim=1).cpu().numpy()
|
| 95 |
-
predictions.extend(preds)
|
| 96 |
-
true_labels.extend(labels.cpu().numpy())
|
| 97 |
-
|
| 98 |
-
acc = accuracy_score(true_labels, predictions)
|
| 99 |
-
f1 = f1_score(true_labels, predictions, average='weighted')
|
| 100 |
-
return acc, f1
|
| 101 |
-
|
| 102 |
-
def train_model(model, train_loader, val_loader, epochs=10): # Increased number of epochs
|
| 103 |
-
for epoch in range(epochs):
|
| 104 |
-
model.train()
|
| 105 |
-
total_loss = 0
|
| 106 |
-
|
| 107 |
-
for batch in tqdm(train_loader, desc=f"Epoch {epoch + 1}/{epochs}"):
|
| 108 |
-
input_ids = batch["input_ids"].to(device)
|
| 109 |
-
attention_mask = batch["attention_mask"].to(device)
|
| 110 |
-
labels = batch["labels"].to(device)
|
| 111 |
-
|
| 112 |
-
optimizer.zero_grad()
|
| 113 |
-
outputs = model(input_ids, attention_mask)
|
| 114 |
-
loss = criterion(outputs, labels)
|
| 115 |
-
loss.backward()
|
| 116 |
-
optimizer.step()
|
| 117 |
-
|
| 118 |
-
total_loss += loss.item()
|
| 119 |
-
|
| 120 |
-
scheduler.step() # Step the scheduler
|
| 121 |
-
|
| 122 |
-
train_acc, train_f1 = evaluate_model(model, train_loader)
|
| 123 |
-
val_acc, val_f1 = evaluate_model(model, val_loader)
|
| 124 |
-
print(f"Epoch {epoch + 1}, Loss: {total_loss / len(train_loader)}, Train Acc: {train_acc:.4f}, Train F1: {train_f1:.4f}, Val Acc: {val_acc:.4f}, Val F1: {val_f1:.4f}")
|
| 125 |
-
|
| 126 |
-
# Save the model after each epoch
|
| 127 |
-
torch.save(model.state_dict(), f"model_epoch_{epoch + 1}.pth")
|
| 128 |
-
|
| 129 |
-
train_model(model, train_loader, val_loader)
|
| 130 |
-
|
| 131 |
-
# Assuming you have a test dataset
|
| 132 |
-
test_texts = dataset["test"]["text"]
|
| 133 |
-
test_labels = fix_labels(dataset["test"]["labels"])
|
| 134 |
-
test_encodings = tokenize(test_texts)
|
| 135 |
-
test_encodings = {key: val.to(device) for key, val in test_encodings.items()}
|
| 136 |
-
test_dataset = EmotionDataset(test_encodings, test_labels)
|
| 137 |
-
test_loader = DataLoader(test_dataset, batch_size=16)
|
| 138 |
-
|
| 139 |
-
test_acc, test_f1 = evaluate_model(model, test_loader)
|
| 140 |
-
print(f"Test Accuracy: {test_acc:.4f}, Test F1 Score: {test_f1:.4f}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|