|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| import warnings
|
| import os
|
| from pathlib import Path
|
| import csv
|
| import json
|
| import torch
|
|
|
| import datasets.rayan_dataset as rayan_dataset
|
| from evaluation.utils.metrics import compute_metrics
|
|
|
| warnings.filterwarnings("ignore")
|
|
|
|
|
| class BaseEval:
|
| def __init__(self, cfg):
|
| self.cfg = cfg
|
| self.device = torch.device(
|
| "cuda:{}".format(cfg["device"]) if torch.cuda.is_available() else "cpu"
|
| )
|
|
|
| self.path = cfg["datasets"]["data_path"]
|
| self.dataset = cfg["datasets"]["dataset_name"]
|
| self.save_csv = cfg["testing"]["save_csv"]
|
| self.save_json = cfg["testing"]["save_json"]
|
| self.categories = cfg["datasets"]["class_name"]
|
| if isinstance(self.categories, str):
|
| if self.categories.lower() == "all":
|
| if self.dataset == "rayan_dataset":
|
| self.categories = self.get_available_class_names(self.path)
|
| else:
|
| self.categories = [self.categories]
|
| self.output_dir = cfg["testing"]["output_dir"]
|
| os.makedirs(self.output_dir, exist_ok=True)
|
| self.scores_dir = cfg["testing"]["output_scores_dir"]
|
| self.class_name_mapping_dir = cfg["testing"]["class_name_mapping_dir"]
|
|
|
| self.leaderboard_metric_weights = {
|
| "image_auroc": 1.2,
|
| "image_ap": 1.1,
|
| "image_f1": 1.1,
|
| "pixel_auroc": 1.0,
|
| "pixel_aupro": 1.4,
|
| "pixel_ap": 1.3,
|
| "pixel_f1": 1.3,
|
| }
|
|
|
| def get_available_class_names(self, root_data_path):
|
| all_items = os.listdir(root_data_path)
|
| folder_names = [
|
| item
|
| for item in all_items
|
| if os.path.isdir(os.path.join(root_data_path, item))
|
| ]
|
|
|
| return folder_names
|
|
|
| def load_datasets(self, category):
|
| dataset_classes = {
|
| "rayan_dataset": rayan_dataset.RayanDataset,
|
| }
|
|
|
| dataset_splits = {
|
| "rayan_dataset": rayan_dataset.DatasetSplit.TEST,
|
| }
|
|
|
| test_dataset = dataset_classes[self.dataset](
|
| source=self.path,
|
| split=dataset_splits[self.dataset],
|
| classname=category,
|
| )
|
| return test_dataset
|
|
|
| def get_category_metrics(self, category):
|
| print(f"Loading scores of '{category}'")
|
| gt_sp, pr_sp, gt_px, pr_px, _ = self.load_category_scores(category)
|
|
|
| print(f"Computing metrics for '{category}'")
|
| image_metric, pixel_metric = compute_metrics(gt_sp, pr_sp, gt_px, pr_px)
|
|
|
| return image_metric, pixel_metric
|
|
|
| def load_category_scores(self, category):
|
| raise NotImplementedError()
|
|
|
| def get_scores_path_for_image(self, image_path):
|
| """example image_path: './data/photovoltaic_module/test/good/037.png'"""
|
| path = Path(image_path)
|
|
|
| category, split, anomaly_type = path.parts[-4:-1]
|
| image_name = path.stem
|
|
|
| return os.path.join(
|
| self.scores_dir, category, split, anomaly_type, f"{image_name}_scores.json"
|
| )
|
|
|
| def calc_leaderboard_score(self, **metrics):
|
| weighted_sum = 0
|
| total_weight = 0
|
| for key, weight in self.leaderboard_metric_weights.items():
|
| metric = metrics.get(key)
|
| weighted_sum += metric * weight
|
| total_weight += weight
|
|
|
| if total_weight == 0:
|
| return 0
|
|
|
| return weighted_sum / total_weight
|
|
|
| def main(self):
|
| image_auroc_list = []
|
| image_f1_list = []
|
| image_ap_list = []
|
| pixel_auroc_list = []
|
| pixel_f1_list = []
|
| pixel_ap_list = []
|
| pixel_aupro_list = []
|
| leaderboard_score_list = []
|
| for category in self.categories:
|
| image_metric, pixel_metric = self.get_category_metrics(
|
| category=category,
|
| )
|
| image_auroc, image_f1, image_ap = image_metric
|
| pixel_auroc, pixel_f1, pixel_ap, pixel_aupro = pixel_metric
|
| leaderboard_score = self.calc_leaderboard_score(
|
| image_auroc=image_auroc,
|
| image_f1=image_f1,
|
| image_ap=image_ap,
|
| pixel_auroc=pixel_auroc,
|
| pixel_aupro=pixel_aupro,
|
| pixel_f1=pixel_f1,
|
| pixel_ap=pixel_ap,
|
| )
|
|
|
| image_auroc_list.append(image_auroc)
|
| image_f1_list.append(image_f1)
|
| image_ap_list.append(image_ap)
|
| pixel_auroc_list.append(pixel_auroc)
|
| pixel_f1_list.append(pixel_f1)
|
| pixel_ap_list.append(pixel_ap)
|
| pixel_aupro_list.append(pixel_aupro)
|
| leaderboard_score_list.append(leaderboard_score)
|
|
|
| print(category)
|
| print(
|
| "[image level] auroc:{}, f1:{}, ap:{}".format(
|
| image_auroc * 100,
|
| image_f1 * 100,
|
| image_ap * 100,
|
| )
|
| )
|
| print(
|
| "[pixel level] auroc:{}, f1:{}, ap:{}, aupro:{}".format(
|
| pixel_auroc * 100,
|
| pixel_f1 * 100,
|
| pixel_ap * 100,
|
| pixel_aupro * 100,
|
| )
|
| )
|
| print(
|
| "leaderboard score:{}".format(
|
| leaderboard_score * 100,
|
| )
|
| )
|
|
|
| image_auroc_mean = sum(image_auroc_list) / len(image_auroc_list)
|
| image_f1_mean = sum(image_f1_list) / len(image_f1_list)
|
| image_ap_mean = sum(image_ap_list) / len(image_ap_list)
|
| pixel_auroc_mean = sum(pixel_auroc_list) / len(pixel_auroc_list)
|
| pixel_f1_mean = sum(pixel_f1_list) / len(pixel_f1_list)
|
| pixel_ap_mean = sum(pixel_ap_list) / len(pixel_ap_list)
|
| pixel_aupro_mean = sum(pixel_aupro_list) / len(pixel_aupro_list)
|
| leaderboard_score_mean = sum(leaderboard_score_list) / len(
|
| leaderboard_score_list
|
| )
|
|
|
| print("mean")
|
| print(
|
| "[image level] auroc:{}, f1:{}, ap:{}".format(
|
| image_auroc_mean * 100, image_f1_mean * 100, image_ap_mean * 100
|
| )
|
| )
|
| print(
|
| "[pixel level] auroc:{}, f1:{}, ap:{}, aupro:{}".format(
|
| pixel_auroc_mean * 100,
|
| pixel_f1_mean * 100,
|
| pixel_ap_mean * 100,
|
| pixel_aupro_mean * 100,
|
| )
|
| )
|
| print(
|
| "leaderboard score:{}".format(
|
| leaderboard_score_mean * 100,
|
| )
|
| )
|
|
|
|
|
| if self.save_csv:
|
| with open(self.class_name_mapping_dir, "r") as f:
|
| class_name_mapping_dict = json.load(f)
|
| csv_data = [
|
| [
|
| "Category",
|
| "pixel_auroc",
|
| "pixel_f1",
|
| "pixel_ap",
|
| "pixel_aupro",
|
| "image_auroc",
|
| "image_f1",
|
| "image_ap",
|
| "leaderboard_score",
|
| ]
|
| ]
|
| for i, category in enumerate(self.categories):
|
| csv_data.append(
|
| [
|
| class_name_mapping_dict[category],
|
| pixel_auroc_list[i] * 100,
|
| pixel_f1_list[i] * 100,
|
| pixel_ap_list[i] * 100,
|
| pixel_aupro_list[i] * 100,
|
| image_auroc_list[i] * 100,
|
| image_f1_list[i] * 100,
|
| image_ap_list[i] * 100,
|
| leaderboard_score_list[i] * 100,
|
| ]
|
| )
|
| csv_data.append(
|
| [
|
| "mean",
|
| pixel_auroc_mean * 100,
|
| pixel_f1_mean * 100,
|
| pixel_ap_mean * 100,
|
| pixel_aupro_mean * 100,
|
| image_auroc_mean * 100,
|
| image_f1_mean * 100,
|
| image_ap_mean * 100,
|
| leaderboard_score_mean * 100,
|
| ]
|
| )
|
|
|
| csv_file_path = os.path.join(self.output_dir, "results.csv")
|
| with open(csv_file_path, mode="w", newline="") as file:
|
| writer = csv.writer(file)
|
| writer.writerows(csv_data)
|
|
|
|
|
| if self.save_json:
|
| json_data = []
|
| with open(self.class_name_mapping_dir, "r") as f:
|
| class_name_mapping_dict = json.load(f)
|
| for i, category in enumerate(self.categories):
|
| json_data.append(
|
| {
|
| "Category": class_name_mapping_dict[category],
|
| "pixel_auroc": pixel_auroc_list[i] * 100,
|
| "pixel_f1": pixel_f1_list[i] * 100,
|
| "pixel_ap": pixel_ap_list[i] * 100,
|
| "pixel_aupro": pixel_aupro_list[i] * 100,
|
| "image_auroc": image_auroc_list[i] * 100,
|
| "image_f1": image_f1_list[i] * 100,
|
| "image_ap": image_ap_list[i] * 100,
|
| "leaderboard_score": leaderboard_score_list[i] * 100,
|
| }
|
| )
|
| json_data.append(
|
| {
|
| "Category": "mean",
|
| "pixel_auroc": pixel_auroc_mean * 100,
|
| "pixel_f1": pixel_f1_mean * 100,
|
| "pixel_ap": pixel_ap_mean * 100,
|
| "pixel_aupro": pixel_aupro_mean * 100,
|
| "image_auroc": image_auroc_mean * 100,
|
| "image_f1": image_f1_mean * 100,
|
| "image_ap": image_ap_mean * 100,
|
| "leaderboard_score": leaderboard_score_mean * 100,
|
| }
|
| )
|
|
|
| json_file_path = os.path.join(self.output_dir, "results.json")
|
| with open(json_file_path, mode="w") as file:
|
| final_json = {
|
| "result": leaderboard_score_mean * 100,
|
| "metadata": json_data,
|
| }
|
| json.dump(final_json, file, indent=4)
|
|
|
