indoml / code_final1_devmodel++.py
Code Files
225a1cb about 3 years ago
28.8 kB
# -*- coding: utf-8 -*-
"""code_final1-devmodel++.ipynb
Automatically generated by Colaboratory.
Original file is located at
    https://colab.research.google.com/drive/1WJYgMTf0rsoiaUtsxV05faaToWunU0a4
"""

!pip install transformers torch
!pip install accelerate -U

from google.colab import files
import zipfile
import os

uploaded = files.upload()

for fn in uploaded.keys():
    print('User uploaded file "{name}" with length {length} bytes'.format(
        name=fn, length=len(uploaded[fn])))

    if fn.endswith('.zip'):
        with zipfile.ZipFile(fn, 'r') as zip_ref:
            zip_ref.extractall()

        print(f"Files from '{fn}' unzipped in the current directory!")
    else:
        print(f"'{fn}' is not a zip file.")

os.listdir()

from transformers import BertTokenizer, BertForSequenceClassification
import torch

# Paths
tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
model_path = '/content/drive/MyDrive/bert_model/model.pth'

# Load Tokenizer
tokenizer = BertTokenizer.from_pretrained(tokenizer_path)

# Load Model
model_state_dict = torch.load(model_path)
model = BertForSequenceClassification.from_pretrained(
    None,
    config='bert-base-multilingual-cased',  # use the correct config
    state_dict=model_state_dict,
    num_labels=len(set(intents))
)

import json
from transformers import BertTokenizer, BertForSequenceClassification, BertConfig
import torch

# Paths to tokenizer and model
tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
model_path = '/content/drive/MyDrive/bert_model/model.pth'

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

# Load Tokenizer
tokenizer = BertTokenizer.from_pretrained(tokenizer_path)

# Initialize model with original number of labels
original_num_labels = 60
config_original = BertConfig.from_pretrained('bert-base-multilingual-cased', num_labels=original_num_labels)
model = BertForSequenceClassification(config=config_original)

# Load saved state dict
model_state_dict = torch.load(model_path)
model.load_state_dict(model_state_dict)

# Update the classifier to new number of labels
new_num_labels = len(set(intents))
model.num_labels = new_num_labels
model.classifier = torch.nn.Linear(model.config.hidden_size, model.num_labels)

import json
from transformers import BertTokenizer, BertForSequenceClassification, BertConfig, TrainingArguments, Trainer
import torch

# Paths to tokenizer and model
tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
model_path = '/content/drive/MyDrive/bert_model/model.pth'

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

# Load Tokenizer
tokenizer = BertTokenizer.from_pretrained(tokenizer_path)

# Initialize model with original number of labels
original_num_labels = 60
config_original = BertConfig.from_pretrained('bert-base-multilingual-cased', num_labels=original_num_labels)
model = BertForSequenceClassification(config=config_original)

# Load saved state dict
model_state_dict = torch.load(model_path)
model.load_state_dict(model_state_dict)

# Update the classifier to the new number of labels
new_num_labels = len(set(intents))
model.num_labels = new_num_labels
model.classifier = torch.nn.Linear(model.config.hidden_size, model.num_labels)

# Create a label_map for mapping intents to unique ids
label_map = {label: i for i, label in enumerate(set(intents))}

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=20,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

import json
from transformers import BertTokenizer, BertForSequenceClassification, BertConfig, TrainingArguments, Trainer
import torch

# Paths to tokenizer and model
tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
model_path = '/content/drive/MyDrive/bert_model/model.pth'

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

# Load Tokenizer
tokenizer = BertTokenizer.from_pretrained(tokenizer_path)

# Initialize model with original number of labels
original_num_labels = 60
config_original = BertConfig.from_pretrained('bert-base-multilingual-cased', num_labels=original_num_labels)
model = BertForSequenceClassification(config=config_original)

# Load saved state dict
model_state_dict = torch.load(model_path)
model.load_state_dict(model_state_dict)

# Update the classifier to the new number of labels
new_num_labels = len(set(intents))
model.num_labels = new_num_labels
model.classifier = torch.nn.Linear(model.config.hidden_size, model.num_labels)

# Create a label_map for mapping intents to unique ids
label_map = {label: i for i, label in enumerate(set(intents))}

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=40,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

import numpy as np

# Load the test data
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]

test_utterances = [item['utt'] for item in test_data]
test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

# Create a dataset for your test encodings
class TestDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])

test_dataset = TestDataset(test_encodings)

# Predict on the test set
predictions = trainer.predict(test_dataset)

predicted_labels = np.argmax(predictions.predictions, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission file
submission_data = [{"indoml_id": int(item["indoml_id"]), "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'massive_test_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip submission.zip {submission_filename}

!pip install transformers torch sentencepiece

import json
import numpy as np
import torch
from transformers import (BertTokenizer, BertForSequenceClassification,
                          RobertaTokenizer, RobertaForSequenceClassification,
                          XLNetTokenizer, XLNetForSequenceClassification,
                          BertConfig, TrainingArguments, Trainer)
import sys
sys.path.append('/usr/local/lib/python3.10/dist-packages')

# Load surprise data and create intent_map
# ... [This part is the same, you can use from the provided code]

# Create a label_map for mapping intents to unique ids
label_map = {label: i for i, label in enumerate(set(intents))}

# Tokenizers and Models
bert_model_name = "bert-large-uncased"
bert_tokenizer = BertTokenizer.from_pretrained(bert_model_name)
bert_model = BertForSequenceClassification.from_pretrained(bert_model_name, num_labels=num_classes)

roberta_model_name = "roberta-large"
roberta_tokenizer = RobertaTokenizer.from_pretrained(roberta_model_name)
roberta_model = RobertaForSequenceClassification.from_pretrained(roberta_model_name, num_labels=num_classes)

xlnet_model_name = "xlnet-large-cased"
xlnet_tokenizer = XLNetTokenizer.from_pretrained(xlnet_model_name)
xlnet_model = XLNetForSequenceClassification.from_pretrained(xlnet_model_name, num_labels=num_classes)

tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
transfer_tokenizer = BertTokenizer.from_pretrained(tokenizer_path)
original_num_labels = 60
config_original = BertConfig.from_pretrained('bert-base-multilingual-cased', num_labels=original_num_labels)
transfer_model = BertForSequenceClassification(config=config_original)
model_path = '/content/drive/MyDrive/bert_model/model.pth'
transfer_model_state_dict = torch.load(model_path)
transfer_model.load_state_dict(transfer_model_state_dict)
transfer_model.num_labels = num_classes
transfer_model.classifier = torch.nn.Linear(transfer_model.config.hidden_size, num_classes)

# ... [Tokenization and Dataset creation is similar to your provided code]

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,  # Adjust this based on your needs
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Set up Trainers for all models
bert_trainer = Trainer(model=bert_model, args=training_args, train_dataset=bert_train_dataset)
roberta_trainer = Trainer(model=roberta_model, args=training_args, train_dataset=roberta_train_dataset)
xlnet_trainer = Trainer(model=xlnet_model, args=training_args, train_dataset=train_dataset)  # For XLNet
transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=train_dataset)  # For Transfer model

# Train all models
bert_trainer.train()
roberta_trainer.train()
xlnet_trainer.train()
transfer_trainer.train()

# ... [Load test data is the same]

# Predict with all models
bert_predictions = bert_trainer.predict(bert_test_dataset)
roberta_predictions = roberta_trainer.predict(roberta_test_dataset)
xlnet_predictions = xlnet_trainer.predict(test_dataset)
transfer_predictions = transfer_trainer.predict(test_dataset)

# Ensemble the predictions
avg_logits = (bert_predictions.predictions + roberta_predictions.predictions + xlnet_predictions.predictions + transfer_predictions.predictions) / 4.0
predicted_labels = np.argmax(avg_logits, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission data and save
submission_filename = 'ensemble_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for indoml_id, intent in zip([item["indoml_id"] for item in test_data], predicted_intents):
        submission_file.write(json.dumps({"indoml_id": indoml_id, "intent": intent}) + '\n')

!zip ensemble_submission.zip {submission_filename}

from google.colab import drive
drive.mount('/content/drive')

!pip install sentencepiece

import json
import numpy as np
import torch
from transformers import (BertTokenizer, BertForSequenceClassification, RobertaTokenizer,
                          RobertaForSequenceClassification, XLNetTokenizer, XLNetForSequenceClassification,
                          TrainingArguments, Trainer, BertConfig)

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]
with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

intent_map = {item['indoml_id']: item['intent'] for item in solutions}
indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]
num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

max_length = 64
train_labels = [label_map[label] for label in intents]

from torch.utils.data import Dataset

class IntentDataset(Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

# BERT
bert_model_name = "bert-large-uncased"
bert_tokenizer = BertTokenizer.from_pretrained(bert_model_name)
bert_model = BertForSequenceClassification.from_pretrained(bert_model_name, num_labels=num_classes)
bert_train_encodings = bert_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
bert_train_dataset = IntentDataset(bert_train_encodings, train_labels)

# RoBERTa
roberta_model_name = "roberta-large"
roberta_tokenizer = RobertaTokenizer.from_pretrained(roberta_model_name)
roberta_model = RobertaForSequenceClassification.from_pretrained(roberta_model_name, num_labels=num_classes)
roberta_train_encodings = roberta_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
roberta_train_dataset = IntentDataset(roberta_train_encodings, train_labels)

# XLNet
xlnet_model_name = "xlnet-large-cased"
xlnet_tokenizer = XLNetTokenizer.from_pretrained(xlnet_model_name)
xlnet_model = XLNetForSequenceClassification.from_pretrained(xlnet_model_name, num_labels=num_classes)
xlnet_train_encodings = xlnet_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
xlnet_train_dataset = IntentDataset(xlnet_train_encodings, train_labels)

# Transfer
model_path = '/content/drive/MyDrive/bert_model/model.pth'
tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
transfer_tokenizer = BertTokenizer(vocab_file='/content/drive/MyDrive/bert_tokenizer/vocab.txt')
config_original = BertConfig.from_pretrained('bert-base-multilingual-cased', num_labels=60)
transfer_model = BertForSequenceClassification(config=config_original)
model_state_dict = torch.load(model_path)
transfer_model.load_state_dict(model_state_dict)
transfer_model.num_labels = num_classes
transfer_model.classifier = torch.nn.Linear(transfer_model.config.hidden_size, num_classes)
transfer_train_encodings = transfer_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
transfer_train_dataset = IntentDataset(transfer_train_encodings, train_labels)

# Common training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=50,
    learning_rate=2e-5,
)

bert_trainer = Trainer(model=bert_model, args=training_args, train_dataset=bert_train_dataset)
rob_trainer = Trainer(model=roberta_model, args=training_args, train_dataset=roberta_train_dataset)
xlnet_trainer = Trainer(model=xlnet_model, args=training_args, train_dataset=xlnet_train_dataset)
transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=transfer_train_dataset)

# Train all models
bert_trainer.train()
rob_trainer.train()
xlnet_trainer.train()

# Common training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=20,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=1000,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=50,
    learning_rate=2e-5,
)

transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=transfer_train_dataset)

# Common training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=50,
    learning_rate=2e-5,
)

rob_trainer = Trainer(model=roberta_model, args=training_args, train_dataset=roberta_train_dataset)
xlnet_trainer = Trainer(model=xlnet_model, args=training_args, train_dataset=xlnet_train_dataset)
transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=transfer_train_dataset)

rob_trainer.train()

xlnet_trainer.train()

# Common training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=20,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=1000,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=50,
    learning_rate=2e-5,
)

transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=transfer_train_dataset)

transfer_trainer.train()

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
bert_model = bert_model.to(device)
roberta_model = roberta_model.to(device)
xlnet_model = xlnet_model.to(device)
transfer_model = transfer_model.to(device)

bert_test_encodings = bert_test_encodings.to(device)
roberta_test_encodings = roberta_test_encodings.to(device)
xlnet_test_encodings = xlnet_test_encodings.to(device)
transfer_test_encodings = transfer_test_encodings.to(device)

# Predict using the trained models
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]
test_utterances = [item['utt'] for item in test_data]

# Encoding test data
bert_test_encodings = bert_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
roberta_test_encodings = roberta_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
xlnet_test_encodings = xlnet_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
transfer_test_encodings = transfer_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

device = torch.device("cpu")
bert_model = bert_model.to(device)
roberta_model = roberta_model.to(device)
xlnet_model = xlnet_model.to(device)
transfer_model = transfer_model.to(device)

bert_test_encodings = bert_test_encodings.to(device)
roberta_test_encodings = roberta_test_encodings.to(device)
xlnet_test_encodings = xlnet_test_encodings.to(device)
transfer_test_encodings = transfer_test_encodings.to(device)

# Get predictions
bert_predictions = bert_model(bert_test_encodings.input_ids, attention_mask=bert_test_encodings.attention_mask).logits
rob_predictions = roberta_model(roberta_test_encodings.input_ids, attention_mask=roberta_test_encodings.attention_mask).logits
xlnet_predictions = xlnet_model(xlnet_test_encodings.input_ids, attention_mask=xlnet_test_encodings.attention_mask).logits
transfer_predictions = transfer_model(transfer_test_encodings.input_ids, attention_mask=transfer_test_encodings.attention_mask).logits

# Predict on the test set using ensemble
avg_predictions = (bert_predictions + rob_predictions + xlnet_predictions + transfer_predictions) / 4.0
predicted_labels = torch.argmax(avg_predictions, dim=1).numpy()
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission file
submission_data = [{"indoml_id": int(item["indoml_id"]), "intent": intent} for item, intent in zip(test_data, predicted_intents)]

submission_filename = 'massive_test_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip submission.zip {submission_filename}

!pip install sentencepiece

import json
import numpy as np
import torch
from transformers import (BertTokenizer, BertForSequenceClassification, RobertaTokenizer,
                          RobertaForSequenceClassification, XLNetTokenizer, XLNetForSequenceClassification,
                          TrainingArguments, Trainer, BertConfig)

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]
with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

intent_map = {item['indoml_id']: item['intent'] for item in solutions}
indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]
num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

max_length = 64
train_labels = [label_map[label] for label in intents]

from torch.utils.data import Dataset

class IntentDataset(Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

# BERT
bert_model_name = "bert-large-uncased"
bert_tokenizer = BertTokenizer.from_pretrained(bert_model_name)
bert_model = BertForSequenceClassification.from_pretrained(bert_model_name, num_labels=num_classes)
bert_train_encodings = bert_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
bert_train_dataset = IntentDataset(bert_train_encodings, train_labels)

# RoBERTa
roberta_model_name = "roberta-large"
roberta_tokenizer = RobertaTokenizer.from_pretrained(roberta_model_name)
roberta_model = RobertaForSequenceClassification.from_pretrained(roberta_model_name, num_labels=num_classes)
roberta_train_encodings = roberta_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
roberta_train_dataset = IntentDataset(roberta_train_encodings, train_labels)

# XLNet
xlnet_model_name = "xlnet-large-cased"
xlnet_tokenizer = XLNetTokenizer.from_pretrained(xlnet_model_name)
xlnet_model = XLNetForSequenceClassification.from_pretrained(xlnet_model_name, num_labels=num_classes)
xlnet_train_encodings = xlnet_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
xlnet_train_dataset = IntentDataset(xlnet_train_encodings, train_labels)

# Transfer
model_path = '/content/drive/MyDrive/bert_model/model.pth'
tokenizer_path = '/content/drive/MyDrive/bert_tokenizer'
transfer_tokenizer = BertTokenizer(vocab_file='/content/drive/MyDrive/bert_tokenizer/vocab.txt')
config_original = BertConfig.from_pretrained('bert-base-multilingual-cased', num_labels=60)
transfer_model = BertForSequenceClassification(config=config_original)
model_state_dict = torch.load(model_path)
transfer_model.load_state_dict(model_state_dict)
transfer_model.num_labels = num_classes
transfer_model.classifier = torch.nn.Linear(transfer_model.config.hidden_size, num_classes)
transfer_train_encodings = transfer_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
transfer_train_dataset = IntentDataset(transfer_train_encodings, train_labels)

# Common training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=50,
    learning_rate=2e-5,
)

bert_trainer = Trainer(model=bert_model, args=training_args, train_dataset=bert_train_dataset)
rob_trainer = Trainer(model=roberta_model, args=training_args, train_dataset=roberta_train_dataset)
xlnet_trainer = Trainer(model=xlnet_model, args=training_args, train_dataset=xlnet_train_dataset)
transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=transfer_train_dataset)

# Train all models
bert_trainer.train()
rob_trainer.train()
xlnet_trainer.train()

# Common training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=20,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=1000,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=50,
    learning_rate=2e-5,
)

transfer_trainer = Trainer(model=transfer_model, args=training_args, train_dataset=transfer_train_dataset)

with open('label_map.json', 'w') as file:
    json.dump(label_map, file)
bert_tokenizer.save_pretrained('./bert_tokenizer_directory')
roberta_tokenizer.save_pretrained('./roberta_tokenizer_directory')
xlnet_tokenizer.save_pretrained('./xlnet_tokenizer_directory')
transfer_tokenizer.save_pretrained('./transfer_tokenizer_directory')

!zip -r models_and_data.zip bert_model bert_tokenizer_directory roberta_model roberta_tokenizer_directory transfer_model transfer_tokenizer_directory xlnet_model xlnet_tokenizer_directory label_map.json