indoml / code1_initial_tests.py
Code Files
225a1cb about 3 years ago
30.9 kB
# -*- coding: utf-8 -*-
"""code1-initial tests.ipynb
Automatically generated by Colaboratory.
Original file is located at
    https://colab.research.google.com/drive/1VE674IhP6sNKQsRiLVd1mLAGKBNGgbIT
"""

!pip install transformers torch
!pip install accelerate -U

from google.colab import files
import zipfile
import os

uploaded = files.upload()

for fn in uploaded.keys():
    print('User uploaded file "{name}" with length {length} bytes'.format(
        name=fn, length=len(uploaded[fn])))

    if fn.endswith('.zip'):
        with zipfile.ZipFile(fn, 'r') as zip_ref:
            zip_ref.extractall()

        print(f"Files from '{fn}' unzipped in the current directory!")
    else:
        print(f"'{fn}' is not a zip file.")

os.listdir()

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

# Split data into train and test sets (2:1) stratified by intent
indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]

utt = [item['utt'] for item in data]

num_classes = len(set(intents))
print("Number of classes")
print(num_classes)

train_data, test_data, train_labels, test_labels = train_test_split(
    utt, intents, test_size=0.33, random_state=42, stratify=intents
)

!pip install transformers torch

# Imports
import json
import random
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import torch
from transformers import BertTokenizer, BertForSequenceClassification, TrainingArguments, Trainer
from sklearn.metrics import f1_score


# Load the data from the JSON files
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

# Split data into train and test sets (2:1) stratified by intent
indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))

train_data, test_data, train_labels, test_labels = train_test_split(
    utt, intents, test_size=0.33, random_state=42, stratify=intents
)

# Use DistilBert for tokenization and model
model_name = "bert-large-uncased"
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Adjust dropout for better regularization
model.config.attention_dropout = 0.5
model.config.seq_classif_dropout = 0.5

# Tokenize the input data
max_length = 64
train_encodings = tokenizer(train_data, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
test_encodings = tokenizer(test_data, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

# Convert labels to numeric values
label_map = {intent: i for i, intent in enumerate(set(train_labels))}
train_labels = [label_map[label] for label in train_labels]
test_labels = [label_map[label] for label in test_labels]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)
test_dataset = IntentDataset(test_encodings, test_labels)

# Adjust training arguments
training_args = TrainingArguments(
    output_dir='./results',
    evaluation_strategy="epoch",
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
    load_best_model_at_end=True,
)
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return {'f1': f1_score(labels, predictions, average='macro')}

# Create the Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=test_dataset,
    compute_metrics=compute_metrics
)

trainer.train()

# Continue with your code for prediction and evaluation...

!pip install transformers torch

# Imports
import json
from transformers import BertTokenizer, BertForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))

# Use BERT-large for tokenization and model
model_name = "bert-large-uncased"
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

# Load the test data
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]

test_utterances = [item['utt'] for item in test_data]
test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

# Create a dataset for your test encodings
class TestDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])

test_dataset = TestDataset(test_encodings)

# Predict on the test set
predictions = trainer.predict(test_dataset)

predicted_labels = np.argmax(predictions.predictions, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission file
submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'massive_test_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip submission.zip {submission_filename}

!pip install transformers torch

# Imports
import json
from transformers import RobertaTokenizer, RobertaForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

# Use RoBERTa-large for tokenization and model
model_name = "roberta-large"
tokenizer = RobertaTokenizer.from_pretrained(model_name)
model = RobertaForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

# Load the test data
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]

test_utterances = [item['utt'] for item in test_data]
test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

# Create a dataset for your test encodings
class TestDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])

test_dataset = TestDataset(test_encodings)

# Predict on the test set
predictions = trainer.predict(test_dataset)

predicted_labels = np.argmax(predictions.predictions, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission file
submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'massive_test_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip submission.zip {submission_filename}

!pip install transformers torch

import json
import numpy as np
import torch
from transformers import BertTokenizer, BertForSequenceClassification, RobertaTokenizer, RobertaForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}
indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]
num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

# Tokenization and Model creation for BERT
bert_model_name = "bert-large-uncased"
bert_tokenizer = BertTokenizer.from_pretrained(bert_model_name)
bert_model = BertForSequenceClassification.from_pretrained(bert_model_name, num_labels=num_classes)

# Tokenization and Model creation for RoBERTa
roberta_model_name = "roberta-large"
roberta_tokenizer = RobertaTokenizer.from_pretrained(roberta_model_name)
roberta_model = RobertaForSequenceClassification.from_pretrained(roberta_model_name, num_labels=num_classes)

# Create dataset
max_length = 64
bert_train_encodings = bert_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
roberta_train_encodings = roberta_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

bert_train_dataset = IntentDataset(bert_train_encodings, train_labels)
roberta_train_dataset = IntentDataset(roberta_train_encodings, train_labels)

# Training arguments and trainer setup
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

bert_trainer = Trainer(model=bert_model, args=training_args, train_dataset=bert_train_dataset)
trainer = Trainer(model=roberta_model, args=training_args, train_dataset=roberta_train_dataset)

# Train both models
bert_trainer.train()

trainer.train()

# Load the test data
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]

test_utterances = [item['utt'] for item in test_data]
bert_test_encodings = bert_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
roberta_test_encodings = roberta_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

class TestDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])

bert_test_dataset = TestDataset(bert_test_encodings)
roberta_test_dataset = TestDataset(roberta_test_encodings)

# Predict with both models
bert_predictions = bert_trainer.predict(bert_test_dataset)
roberta_predictions = trainer.predict(roberta_test_dataset)

# ... [Continuing from previous code]

# Ensemble the predictions
avg_logits = (bert_predictions.predictions + roberta_predictions.predictions) / 2.0

# Choose the classes with the highest logits
predicted_labels = np.argmax(avg_logits, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission data
submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'ensemble_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip ensemble_submission.zip {submission_filename}

# ... [Continuing from previous code]

# Ensemble the predictions
avg_logits = (bert_predictions.predictions + roberta_predictions.predictions) / 2.0

# Choose the classes with the highest logits
predicted_labels = np.argmax(avg_logits, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Ensure that 'indoml_id' is an integer
for item in test_data:
    item["indoml_id"] = int(item["indoml_id"])

# Create submission data
submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'massive_test_shivam.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip ensemble_submission.zip {submission_filename}

!pip install sentencepiece
import sentencepiece
import sys
sys.path.append('/usr/local/lib/python3.10/dist-packages')

# Imports
import json
import torch
import numpy as np
from transformers import XLNetTokenizer, XLNetForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

# Use XLNet-large-cased for tokenization and model
model_name = "xlnet-large-cased"
tokenizer = XLNetTokenizer.from_pretrained(model_name)
model = XLNetForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

# Load the test data
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]

test_utterances = [item['utt'] for item in test_data]
test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

# Create a dataset for your test encodings
class TestDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])

test_dataset = TestDataset(test_encodings)

# Predict on the test set
predictions = trainer.predict(test_dataset)

predicted_labels = np.argmax(predictions.predictions, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission file
submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'massive_test_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip submission.zip {submission_filename}

# Imports
import json
import torch
from transformers import DebertaTokenizer, DebertaForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))

from transformers import DebertaV2Tokenizer, DebertaV2ForSequenceClassification


model_name = "microsoft/deberta-xxlarge-v2"
tokenizer = DebertaV2Tokenizer.from_pretrained(model_name)
model = DebertaV2ForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)




# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [intent_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

# Load the test data
with open('massive_test.data', 'r') as test_file:
    test_data = [json.loads(line) for line in test_file]

test_utterances = [item['utt'] for item in test_data]
test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt')

# Create a dataset for your test encodings
class TestDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: val[idx] for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings["input_ids"])

test_dataset = TestDataset(test_encodings)

# Predict on the test set
predictions = trainer.predict(test_dataset)

predicted_labels = np.argmax(predictions.predictions, axis=1)
predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels]

# Create submission file
submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)]

# Save the submission file
submission_filename = 'massive_test_submission.predict'
with open(submission_filename, 'w') as submission_file:
    for item in submission_data:
        submission_file.write(json.dumps(item) + '\n')

# Zip the submission file
!zip submission.zip {submission_filename}

!pip install transformers torch

# Imports
import json
import torch
from transformers import DebertaTokenizer, DebertaForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

# Use DeBERTa-xxlarge for tokenization and model
model_name = "microsoft/deberta-xxlarge"
tokenizer = DebertaTokenizer.from_pretrained(model_name)
model = DebertaForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

!pip install transformers torch

# Imports
import json
import torch
from transformers import DebertaTokenizer, DebertaForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

# Use DeBERTa-xlarge for tokenization and model
model_name = "microsoft/deberta-xlarge"
tokenizer = DebertaTokenizer.from_pretrained(model_name)
model = DebertaForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Tokenize the input data for training
max_length = 64
train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()

!pip install sentencepiece
import sentencepiece
import sys
sys.path.append('/usr/local/lib/python3.10/dist-packages')

# Imports
import json
import torch
from transformers import T5Tokenizer, T5ForSequenceClassification, TrainingArguments, Trainer

# Load surprise data
with open('surprise.data', 'r') as data_file:
    data = [json.loads(line) for line in data_file]

with open('surprise.solution', 'r') as solution_file:
    solutions = [json.loads(line) for line in solution_file]

# Create a dictionary to map indoml_id to intents
intent_map = {item['indoml_id']: item['intent'] for item in solutions}

indoml_ids = [item['indoml_id'] for item in data]
intents = [intent_map[indoml_id] for indoml_id in indoml_ids]
utt = [item['utt'] for item in data]

num_classes = len(set(intents))
label_map = {intent: i for i, intent in enumerate(set(intents))}

# Use T5-large for tokenization and model
model_name = "t5-large"
tokenizer = T5Tokenizer.from_pretrained(model_name)
model = T5ForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)

# Tokenize the input data for training. Note that T5 prefers the "summarize: " prefix for classification tasks.
max_length = 64
train_encodings = tokenizer(["summarize: " + text for text in utt], truncation=True, padding=True, max_length=max_length, return_tensors='pt')
train_labels = [label_map[label] for label in intents]

# Dataset definition
class IntentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = IntentDataset(train_encodings, train_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    save_strategy="epoch",
    num_train_epochs=10,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
    learning_rate=2e-5,
)

# Train the model
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)
trainer.train()