Harald Deutschmann commited on
Commit
6f73d05
·
1 Parent(s): 566a274
Files changed (1) hide show
  1. train/ocrtrainer.py +4 -3
train/ocrtrainer.py CHANGED
@@ -70,9 +70,10 @@ class OcrTrainer:
70
  def train(self, ner_formats_list, eval_steps):
71
  annotations_as_ner = list(map(normalize_ner_annotation_for_layoutlm, ner_formats_list))
72
  dataset = Dataset.from_list(annotations_as_ner).cast_column("image", Image())
 
73
  column_names = dataset.column_names
74
- dataset = dataset.train_test_split(test_size=0.1)
75
-
76
  features = Features({
77
  'pixel_values': Array3D(dtype="float32", shape=(3, 224, 224)),
78
  'input_ids': Sequence(feature=Value(dtype='int64')),
@@ -96,7 +97,7 @@ class OcrTrainer:
96
 
97
  train_dataset = self.upsample_dataset(train_dataset)
98
 
99
- print("TRAIN DATASET", train_dataset)
100
 
101
  compute_eval_metrics = generate_layoutlm_compute_eval_metric_fn(
102
  ner_labels=self.label_list,
 
70
  def train(self, ner_formats_list, eval_steps):
71
  annotations_as_ner = list(map(normalize_ner_annotation_for_layoutlm, ner_formats_list))
72
  dataset = Dataset.from_list(annotations_as_ner).cast_column("image", Image())
73
+ print("DATASET FETCHED")
74
  column_names = dataset.column_names
75
+ dataset = dataset.train_test_split(test_size=0.2)
76
+ print("DATASET GENERATED")
77
  features = Features({
78
  'pixel_values': Array3D(dtype="float32", shape=(3, 224, 224)),
79
  'input_ids': Sequence(feature=Value(dtype='int64')),
 
97
 
98
  train_dataset = self.upsample_dataset(train_dataset)
99
 
100
+ print("TRAIN DATASET UPSAMPLED", train_dataset)
101
 
102
  compute_eval_metrics = generate_layoutlm_compute_eval_metric_fn(
103
  ner_labels=self.label_list,