import os import numpy as np from docai.training import generate_layoutlm_compute_eval_metric_fn from transformers.data.data_collator import default_data_collator from transformers import AutoProcessor, Trainer, LayoutLMv3ForTokenClassification, TrainingArguments from datasets import Dataset, Features, Array3D, Sequence, Value, Array2D, Image from docai.annotations import normalize_ner_annotation_for_layoutlm import torch from accelerate import Accelerator, DataLoaderConfiguration #from imblearn.over_sampling import SMOTE DEVICE = "cuda" if torch.cuda.is_available() else "cpu" PRETRAINED_MODEL_PATH = "microsoft/layoutlmv3-base" BATCH_SIZE = 64 NER_TAGS = [ "B-Manufacturer", "I-Manufacturer", "B-Delivery_Number", "I-Delivery_Number", "B-Date", "I-Date", "O", ] MODEL_NAME = "./us_dl_model" api_token = os.environ.get("HF_TOKEN") model_path = os.environ.get("HF_MODEL_PATH") train_model_path = os.environ.get("HF_TRAIN_MODEL_PATH") class OcrTrainer: def __init__(self): super().__init__() self.processor = AutoProcessor.from_pretrained(PRETRAINED_MODEL_PATH, apply_ocr=False) self.label_list = NER_TAGS self.id2label = {k: v for k, v in enumerate(self.label_list)} self.label2id = {v: k for k, v in enumerate(self.label_list)} self.accelerator = Accelerator() self.data_loader_config = DataLoaderConfiguration( dispatch_batches=None, # Set as needed split_batches=False, # Set as needed even_batches=True, # Set as needed use_seedable_sampler=True # Set as needed ) #self.data_loader_config = DataLoaderConfiguration(batch_size=8, shuffle=True) self.accelerator.data_loader_config = self.data_loader_config #def upsample_dataset(self, dataset: Dataset, label_col: str = 'labels'): # '''Upsample a dataset without the stream mechanism of HF # ''' # # X contains all the data (labels too). Makes it easier later # X = np.array([ # dataset[col_name] # for col_name in dataset.column_names # ]).T # # Get labels from dataset # y = dataset[label_col] # # OverSample dataset # sm = SMOTE(sampling_strategy='minority') # X_balanced, _ = sm.fit_resample(X, y) # # Return new balanced dataset # return Dataset.from_dict({ # col_name: X_balanced.T[i] # for i, col_name in enumerate(dataset.column_names) # }) def train(self, ner_formats_list, eval_steps): annotations_as_ner = list(map(normalize_ner_annotation_for_layoutlm, ner_formats_list)) dataset = Dataset.from_list(annotations_as_ner).cast_column("image", Image()) print("DATASET FETCHED") column_names = dataset.column_names dataset = dataset.train_test_split(test_size=0.2) print("DATASET GENERATED") features = Features({ 'pixel_values': Array3D(dtype="float32", shape=(3, 224, 224)), 'input_ids': Sequence(feature=Value(dtype='int64')), 'attention_mask': Sequence(Value(dtype='int64')), 'bbox': Array2D(dtype="int64", shape=(512, 4)), 'labels': Sequence(feature=Value(dtype='int64')), }) train_dataset = dataset["train"].map( self.prepare_dataset, batched=True, remove_columns=column_names, features=features, ) eval_dataset = dataset["test"].map( self.prepare_dataset, batched=True, remove_columns=column_names, features=features, ) # train_dataset = self.upsample_dataset(train_dataset) print("TRAIN DATASET", train_dataset) compute_eval_metrics = generate_layoutlm_compute_eval_metric_fn( ner_labels=self.label_list, metric_name="seqeval", return_entity_level_metrics=False ) used_model = model_path if not train_model_path else train_model_path print("USED MODEL", used_model) model = LayoutLMv3ForTokenClassification.from_pretrained( used_model, id2label=self.id2label, label2id=self.label2id, token=api_token) model = self.accelerator.prepare(model) training_args = TrainingArguments( output_dir=MODEL_NAME, max_steps=eval_steps * 10, per_device_train_batch_size=BATCH_SIZE, per_device_eval_batch_size=BATCH_SIZE, learning_rate=2e-5, evaluation_strategy="steps", eval_steps=eval_steps, load_best_model_at_end=True, metric_for_best_model="f1") # Initialize our Trainer trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=self.processor, data_collator=default_data_collator, compute_metrics=compute_eval_metrics ) summary = trainer.train() trainer.evaluate() model.push_to_hub(repo_id=model_path, token=api_token) self.processor.push_to_hub(repo_id=model_path, token=api_token) return summary def prepare_dataset(self, annotations): images = annotations['image'] words = annotations['tokens'] boxes = annotations['bboxes'] # Map over labels and convert to numeric id for each ner_tag ner_tags = [self.convert_ner_tags_to_id(ner_tags) for ner_tags in annotations['ner_tags']] encoding = self.processor(images, words, boxes=boxes, word_labels=ner_tags, truncation=True, padding="max_length") return encoding def convert_ner_tags_to_id(self, ner_tags): return [self.label2id[ner_tag] for ner_tag in ner_tags]