Spaces:
Sleeping
Sleeping
File size: 2,505 Bytes
2d3dd8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | from __future__ import annotations
import argparse
from pathlib import Path
from bayes_gp_llmops.evaluation.pipeline import run_evaluation_pipeline
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Evaluate a trained AG News classifier checkpoint."
)
parser.add_argument("--data-config", type=Path, default=Path("configs/data.yaml"))
parser.add_argument("--model-config", type=Path, default=Path("configs/model.yaml"))
parser.add_argument("--train-config", type=Path, default=Path("configs/train.yaml"))
parser.add_argument("--checkpoint", type=Path, default=None)
parser.add_argument("--device", type=str, default=None)
parser.add_argument("--output-dir", type=Path, default=Path("artifacts/evaluation"))
parser.add_argument("--disable-temperature-scaling", action="store_true")
parser.add_argument("--debug", action="store_true")
return parser.parse_args()
def main() -> None:
args = parse_args()
artifacts = run_evaluation_pipeline(
data_config_path=args.data_config,
model_config_path=args.model_config,
train_config_path=args.train_config,
checkpoint_path=args.checkpoint,
device_override=args.device,
output_dir=args.output_dir,
enable_temperature_scaling=not args.disable_temperature_scaling,
debug_mode=args.debug,
)
print(f"evaluation_output_dir={artifacts.output_dir}")
print(f"metrics_validation={artifacts.metrics_validation_path}")
print(f"metrics_test={artifacts.metrics_test_path}")
if artifacts.metrics_validation_calibrated_path is not None:
print(f"metrics_validation_calibrated={artifacts.metrics_validation_calibrated_path}")
if artifacts.metrics_test_calibrated_path is not None:
print(f"metrics_test_calibrated={artifacts.metrics_test_calibrated_path}")
print(f"predictions_validation={artifacts.predictions_validation_path}")
print(f"predictions_test={artifacts.predictions_test_path}")
print(f"temperature_scaling={artifacts.temperature_scaling_path}")
print(f"confusion_matrix_plot={artifacts.confusion_matrix_plot_path}")
print(f"reliability_diagram_plot={artifacts.reliability_diagram_plot_path}")
print(f"confidence_histogram_plot={artifacts.confidence_histogram_plot_path}")
print(f"entropy_histogram_plot={artifacts.entropy_histogram_plot_path}")
if __name__ == "__main__":
main()
|