# -*- coding: utf-8 -*- """code1-initial tests.ipynb Automatically generated by Colaboratory. Original file is located at https://colab.research.google.com/drive/1VE674IhP6sNKQsRiLVd1mLAGKBNGgbIT """ !pip install transformers torch !pip install accelerate -U from google.colab import files import zipfile import os uploaded = files.upload() for fn in uploaded.keys(): print('User uploaded file "{name}" with length {length} bytes'.format( name=fn, length=len(uploaded[fn]))) if fn.endswith('.zip'): with zipfile.ZipFile(fn, 'r') as zip_ref: zip_ref.extractall() print(f"Files from '{fn}' unzipped in the current directory!") else: print(f"'{fn}' is not a zip file.") os.listdir() # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} # Split data into train and test sets (2:1) stratified by intent indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) print("Number of classes") print(num_classes) train_data, test_data, train_labels, test_labels = train_test_split( utt, intents, test_size=0.33, random_state=42, stratify=intents ) !pip install transformers torch # Imports import json import random import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report import torch from transformers import BertTokenizer, BertForSequenceClassification, TrainingArguments, Trainer from sklearn.metrics import f1_score # Load the data from the JSON files with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} # Split data into train and test sets (2:1) stratified by intent indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) train_data, test_data, train_labels, test_labels = train_test_split( utt, intents, test_size=0.33, random_state=42, stratify=intents ) # Use DistilBert for tokenization and model model_name = "bert-large-uncased" tokenizer = BertTokenizer.from_pretrained(model_name) model = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Adjust dropout for better regularization model.config.attention_dropout = 0.5 model.config.seq_classif_dropout = 0.5 # Tokenize the input data max_length = 64 train_encodings = tokenizer(train_data, truncation=True, padding=True, max_length=max_length, return_tensors='pt') test_encodings = tokenizer(test_data, truncation=True, padding=True, max_length=max_length, return_tensors='pt') # Convert labels to numeric values label_map = {intent: i for i, intent in enumerate(set(train_labels))} train_labels = [label_map[label] for label in train_labels] test_labels = [label_map[label] for label in test_labels] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) test_dataset = IntentDataset(test_encodings, test_labels) # Adjust training arguments training_args = TrainingArguments( output_dir='./results', evaluation_strategy="epoch", save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, load_best_model_at_end=True, ) def compute_metrics(eval_pred): logits, labels = eval_pred predictions = np.argmax(logits, axis=-1) return {'f1': f1_score(labels, predictions, average='macro')} # Create the Trainer trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=test_dataset, compute_metrics=compute_metrics ) trainer.train() # Continue with your code for prediction and evaluation... !pip install transformers torch # Imports import json from transformers import BertTokenizer, BertForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) # Use BERT-large for tokenization and model model_name = "bert-large-uncased" tokenizer = BertTokenizer.from_pretrained(model_name) model = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training max_length = 64 train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() # Load the test data with open('massive_test.data', 'r') as test_file: test_data = [json.loads(line) for line in test_file] test_utterances = [item['utt'] for item in test_data] test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt') # Create a dataset for your test encodings class TestDataset(torch.utils.data.Dataset): def __init__(self, encodings): self.encodings = encodings def __getitem__(self, idx): return {key: val[idx] for key, val in self.encodings.items()} def __len__(self): return len(self.encodings["input_ids"]) test_dataset = TestDataset(test_encodings) # Predict on the test set predictions = trainer.predict(test_dataset) predicted_labels = np.argmax(predictions.predictions, axis=1) predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels] # Create submission file submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)] # Save the submission file submission_filename = 'massive_test_submission.predict' with open(submission_filename, 'w') as submission_file: for item in submission_data: submission_file.write(json.dumps(item) + '\n') # Zip the submission file !zip submission.zip {submission_filename} !pip install transformers torch # Imports import json from transformers import RobertaTokenizer, RobertaForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) label_map = {intent: i for i, intent in enumerate(set(intents))} # Use RoBERTa-large for tokenization and model model_name = "roberta-large" tokenizer = RobertaTokenizer.from_pretrained(model_name) model = RobertaForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training max_length = 64 train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() # Load the test data with open('massive_test.data', 'r') as test_file: test_data = [json.loads(line) for line in test_file] test_utterances = [item['utt'] for item in test_data] test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt') # Create a dataset for your test encodings class TestDataset(torch.utils.data.Dataset): def __init__(self, encodings): self.encodings = encodings def __getitem__(self, idx): return {key: val[idx] for key, val in self.encodings.items()} def __len__(self): return len(self.encodings["input_ids"]) test_dataset = TestDataset(test_encodings) # Predict on the test set predictions = trainer.predict(test_dataset) predicted_labels = np.argmax(predictions.predictions, axis=1) predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels] # Create submission file submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)] # Save the submission file submission_filename = 'massive_test_submission.predict' with open(submission_filename, 'w') as submission_file: for item in submission_data: submission_file.write(json.dumps(item) + '\n') # Zip the submission file !zip submission.zip {submission_filename} !pip install transformers torch import json import numpy as np import torch from transformers import BertTokenizer, BertForSequenceClassification, RobertaTokenizer, RobertaForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) label_map = {intent: i for i, intent in enumerate(set(intents))} # Tokenization and Model creation for BERT bert_model_name = "bert-large-uncased" bert_tokenizer = BertTokenizer.from_pretrained(bert_model_name) bert_model = BertForSequenceClassification.from_pretrained(bert_model_name, num_labels=num_classes) # Tokenization and Model creation for RoBERTa roberta_model_name = "roberta-large" roberta_tokenizer = RobertaTokenizer.from_pretrained(roberta_model_name) roberta_model = RobertaForSequenceClassification.from_pretrained(roberta_model_name, num_labels=num_classes) # Create dataset max_length = 64 bert_train_encodings = bert_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') roberta_train_encodings = roberta_tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) bert_train_dataset = IntentDataset(bert_train_encodings, train_labels) roberta_train_dataset = IntentDataset(roberta_train_encodings, train_labels) # Training arguments and trainer setup training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) bert_trainer = Trainer(model=bert_model, args=training_args, train_dataset=bert_train_dataset) trainer = Trainer(model=roberta_model, args=training_args, train_dataset=roberta_train_dataset) # Train both models bert_trainer.train() trainer.train() # Load the test data with open('massive_test.data', 'r') as test_file: test_data = [json.loads(line) for line in test_file] test_utterances = [item['utt'] for item in test_data] bert_test_encodings = bert_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt') roberta_test_encodings = roberta_tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt') class TestDataset(torch.utils.data.Dataset): def __init__(self, encodings): self.encodings = encodings def __getitem__(self, idx): return {key: val[idx] for key, val in self.encodings.items()} def __len__(self): return len(self.encodings["input_ids"]) bert_test_dataset = TestDataset(bert_test_encodings) roberta_test_dataset = TestDataset(roberta_test_encodings) # Predict with both models bert_predictions = bert_trainer.predict(bert_test_dataset) roberta_predictions = trainer.predict(roberta_test_dataset) # ... [Continuing from previous code] # Ensemble the predictions avg_logits = (bert_predictions.predictions + roberta_predictions.predictions) / 2.0 # Choose the classes with the highest logits predicted_labels = np.argmax(avg_logits, axis=1) predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels] # Create submission data submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)] # Save the submission file submission_filename = 'ensemble_submission.predict' with open(submission_filename, 'w') as submission_file: for item in submission_data: submission_file.write(json.dumps(item) + '\n') # Zip the submission file !zip ensemble_submission.zip {submission_filename} # ... [Continuing from previous code] # Ensemble the predictions avg_logits = (bert_predictions.predictions + roberta_predictions.predictions) / 2.0 # Choose the classes with the highest logits predicted_labels = np.argmax(avg_logits, axis=1) predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels] # Ensure that 'indoml_id' is an integer for item in test_data: item["indoml_id"] = int(item["indoml_id"]) # Create submission data submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)] # Save the submission file submission_filename = 'massive_test_shivam.predict' with open(submission_filename, 'w') as submission_file: for item in submission_data: submission_file.write(json.dumps(item) + '\n') # Zip the submission file !zip ensemble_submission.zip {submission_filename} !pip install sentencepiece import sentencepiece import sys sys.path.append('/usr/local/lib/python3.10/dist-packages') # Imports import json import torch import numpy as np from transformers import XLNetTokenizer, XLNetForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) label_map = {intent: i for i, intent in enumerate(set(intents))} # Use XLNet-large-cased for tokenization and model model_name = "xlnet-large-cased" tokenizer = XLNetTokenizer.from_pretrained(model_name) model = XLNetForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training max_length = 64 train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() # Load the test data with open('massive_test.data', 'r') as test_file: test_data = [json.loads(line) for line in test_file] test_utterances = [item['utt'] for item in test_data] test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt') # Create a dataset for your test encodings class TestDataset(torch.utils.data.Dataset): def __init__(self, encodings): self.encodings = encodings def __getitem__(self, idx): return {key: val[idx] for key, val in self.encodings.items()} def __len__(self): return len(self.encodings["input_ids"]) test_dataset = TestDataset(test_encodings) # Predict on the test set predictions = trainer.predict(test_dataset) predicted_labels = np.argmax(predictions.predictions, axis=1) predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels] # Create submission file submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)] # Save the submission file submission_filename = 'massive_test_submission.predict' with open(submission_filename, 'w') as submission_file: for item in submission_data: submission_file.write(json.dumps(item) + '\n') # Zip the submission file !zip submission.zip {submission_filename} # Imports import json import torch from transformers import DebertaTokenizer, DebertaForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) from transformers import DebertaV2Tokenizer, DebertaV2ForSequenceClassification model_name = "microsoft/deberta-xxlarge-v2" tokenizer = DebertaV2Tokenizer.from_pretrained(model_name) model = DebertaV2ForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training max_length = 64 train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [intent_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() # Load the test data with open('massive_test.data', 'r') as test_file: test_data = [json.loads(line) for line in test_file] test_utterances = [item['utt'] for item in test_data] test_encodings = tokenizer(test_utterances, truncation=True, padding=True, max_length=max_length, return_tensors='pt') # Create a dataset for your test encodings class TestDataset(torch.utils.data.Dataset): def __init__(self, encodings): self.encodings = encodings def __getitem__(self, idx): return {key: val[idx] for key, val in self.encodings.items()} def __len__(self): return len(self.encodings["input_ids"]) test_dataset = TestDataset(test_encodings) # Predict on the test set predictions = trainer.predict(test_dataset) predicted_labels = np.argmax(predictions.predictions, axis=1) predicted_intents = [list(label_map.keys())[list(label_map.values()).index(label)] for label in predicted_labels] # Create submission file submission_data = [{"indoml_id": item["indoml_id"], "intent": intent} for item, intent in zip(test_data, predicted_intents)] # Save the submission file submission_filename = 'massive_test_submission.predict' with open(submission_filename, 'w') as submission_file: for item in submission_data: submission_file.write(json.dumps(item) + '\n') # Zip the submission file !zip submission.zip {submission_filename} !pip install transformers torch # Imports import json import torch from transformers import DebertaTokenizer, DebertaForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) label_map = {intent: i for i, intent in enumerate(set(intents))} # Use DeBERTa-xxlarge for tokenization and model model_name = "microsoft/deberta-xxlarge" tokenizer = DebertaTokenizer.from_pretrained(model_name) model = DebertaForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training max_length = 64 train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() !pip install transformers torch # Imports import json import torch from transformers import DebertaTokenizer, DebertaForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) label_map = {intent: i for i, intent in enumerate(set(intents))} # Use DeBERTa-xlarge for tokenization and model model_name = "microsoft/deberta-xlarge" tokenizer = DebertaTokenizer.from_pretrained(model_name) model = DebertaForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training max_length = 64 train_encodings = tokenizer(utt, truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() !pip install sentencepiece import sentencepiece import sys sys.path.append('/usr/local/lib/python3.10/dist-packages') # Imports import json import torch from transformers import T5Tokenizer, T5ForSequenceClassification, TrainingArguments, Trainer # Load surprise data with open('surprise.data', 'r') as data_file: data = [json.loads(line) for line in data_file] with open('surprise.solution', 'r') as solution_file: solutions = [json.loads(line) for line in solution_file] # Create a dictionary to map indoml_id to intents intent_map = {item['indoml_id']: item['intent'] for item in solutions} indoml_ids = [item['indoml_id'] for item in data] intents = [intent_map[indoml_id] for indoml_id in indoml_ids] utt = [item['utt'] for item in data] num_classes = len(set(intents)) label_map = {intent: i for i, intent in enumerate(set(intents))} # Use T5-large for tokenization and model model_name = "t5-large" tokenizer = T5Tokenizer.from_pretrained(model_name) model = T5ForSequenceClassification.from_pretrained(model_name, num_labels=num_classes) # Tokenize the input data for training. Note that T5 prefers the "summarize: " prefix for classification tasks. max_length = 64 train_encodings = tokenizer(["summarize: " + text for text in utt], truncation=True, padding=True, max_length=max_length, return_tensors='pt') train_labels = [label_map[label] for label in intents] # Dataset definition class IntentDataset(torch.utils.data.Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: val[idx] for key, val in self.encodings.items()} item['labels'] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) train_dataset = IntentDataset(train_encodings, train_labels) # Training arguments training_args = TrainingArguments( output_dir='./results', save_strategy="epoch", num_train_epochs=10, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir='./logs', logging_steps=10, learning_rate=2e-5, ) # Train the model trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train()