MRC001 / trainer_qa.py

Add my custom Trainer for Question Answering Problem

f0f242a about 4 years ago

3.54 kB

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput

class QuestionAnsweringTrainer(Trainer):
    def __init__(self, *args, post_process_function = None, **kwargs):
        super().__init__(*args, **kwargs)
        self.post_process_function = post_process_function

    def evaluate(self, eval_dataset = None, ignore_keys = None, metric_key_prefix: str = "eval"):
        eval_dataset    = self.eval_dataset if eval_dataset is None else eval_dataset
        eval_dataloader = self.get_eval_dataloader(eval_dataset)

        # Temporarily disable metric computation, we will do it in the loop here.
        compute_metrics = self.compute_metrics
        self.compute_metrics = None
        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
            output = eval_loop(
                eval_dataloader,
                description="Evaluation",
                # No point gathering the predictions if there are no metrics, otherwise we defer to
                # self.args.prediction_loss_only
                prediction_loss_only=True if compute_metrics is None else None,
                ignore_keys=ignore_keys,
            )
        finally:
            self.compute_metrics = compute_metrics

        if self.post_process_function is not None and self.compute_metrics is not None:
            eval_preds = self.post_process_function(eval_dataset, self.tokenizer, output.predictions)
            metrics = self.compute_metrics(eval_preds)

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()):
                if not key.startswith(f"{metric_key_prefix}_"):
                    metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)

            self.log(metrics)
        else:
            metrics = {}

        self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)
        return metrics

    def predict(self, predict_dataset, ignore_keys=None, metric_key_prefix: str = "test"):
        predict_dataloader = self.get_test_dataloader(predict_dataset)

        # Temporarily disable metric computation, we will do it in the loop here.
        compute_metrics = self.compute_metrics
        self.compute_metrics = None
        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
            output = eval_loop(
                predict_dataloader,
                description="Prediction",
                # No point gathering the predictions if there are no metrics, otherwise we defer to
                # self.args.prediction_loss_only
                prediction_loss_only=True if compute_metrics is None else None,
                ignore_keys=ignore_keys,
            )
        finally:
            self.compute_metrics = compute_metrics

        if self.post_process_function is None or self.compute_metrics is None:
            return output

        predictions = self.post_process_function(predict_dataset, self.tokenizer, output.predictions, "predict")
        metrics = self.compute_metrics(predictions)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()):
            if not key.startswith(f"{metric_key_prefix}_"):
                metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)

        return  PredictionOutput(predictions=predictions.predictions, label_ids=predictions.label_ids, metrics=metrics)

	from transformers import Trainer, is_torch_tpu_available
	from transformers.trainer_utils import PredictionOutput

	class QuestionAnsweringTrainer(Trainer):
	def __init__(self, args, post_process_function = None, *kwargs):
	super().__init__(args, *kwargs)
	self.post_process_function = post_process_function

	def evaluate(self, eval_dataset = None, ignore_keys = None, metric_key_prefix: str = "eval"):
	eval_dataset = self.eval_dataset if eval_dataset is None else eval_dataset
	eval_dataloader = self.get_eval_dataloader(eval_dataset)

	# Temporarily disable metric computation, we will do it in the loop here.
	compute_metrics = self.compute_metrics
	self.compute_metrics = None
	eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
	try:
	output = eval_loop(
	eval_dataloader,
	description="Evaluation",
	# No point gathering the predictions if there are no metrics, otherwise we defer to
	# self.args.prediction_loss_only
	prediction_loss_only=True if compute_metrics is None else None,
	ignore_keys=ignore_keys,
	)
	finally:
	self.compute_metrics = compute_metrics

	if self.post_process_function is not None and self.compute_metrics is not None:
	eval_preds = self.post_process_function(eval_dataset, self.tokenizer, output.predictions)
	metrics = self.compute_metrics(eval_preds)

	# Prefix all keys with metric_key_prefix + '_'
	for key in list(metrics.keys()):
	if not key.startswith(f"{metric_key_prefix}_"):
	metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)

	self.log(metrics)
	else:
	metrics = {}

	self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)
	return metrics

	def predict(self, predict_dataset, ignore_keys=None, metric_key_prefix: str = "test"):
	predict_dataloader = self.get_test_dataloader(predict_dataset)

	# Temporarily disable metric computation, we will do it in the loop here.
	compute_metrics = self.compute_metrics
	self.compute_metrics = None
	eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
	try:
	output = eval_loop(
	predict_dataloader,
	description="Prediction",
	# No point gathering the predictions if there are no metrics, otherwise we defer to
	# self.args.prediction_loss_only
	prediction_loss_only=True if compute_metrics is None else None,
	ignore_keys=ignore_keys,
	)
	finally:
	self.compute_metrics = compute_metrics

	if self.post_process_function is None or self.compute_metrics is None:
	return output

	predictions = self.post_process_function(predict_dataset, self.tokenizer, output.predictions, "predict")
	metrics = self.compute_metrics(predictions)

	# Prefix all keys with metric_key_prefix + '_'
	for key in list(metrics.keys()):
	if not key.startswith(f"{metric_key_prefix}_"):
	metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)

	return PredictionOutput(predictions=predictions.predictions, label_ids=predictions.label_ids, metrics=metrics)