scripts/eval_ius.py

import os
import torch
import argparse
import numpy as np
import pandas as pd

from pathlib import Path
from datetime import datetime

# mine
from ius.ius import IUS
from ius.ius_eval_parser import IUSEvalParser
from data.loading import IUSEvalDataset
from data.dataloader import to_dataloader
from utils.eval_utils import EPUCNNEval
from utils.train_utils import create_output_folders, save_to_json
from utils.omega_parser import IUSConfig


BASE_PATH = Path(__file__).resolve().parent


def parse_options():
    parser = argparse.ArgumentParser()
    parser.add_argument("--experiment_folder_name", type=str, required=True,
                        help="Folder name containing epu configuration & saved ckpt")
    parser.add_argument("--cb_vector_tag", type=str, default=None, required=True,
                        help="cb vector data. If not specified estimates all cb_vectors")
    parser.add_argument("--synthetic_images", type=str, default='png',
                        help="It can be either a single synthetic image file or a folder path containing multiple "
                             "synthetic images")
    parser.add_argument("--synthetic_img_extension", type=str, default='png',
                        help="Extension of synthetic images")
    args = parser.parse_args()
    return args


def main():
    args = parse_options()
    cb_vec_tag = args.cb_vector_tag
    synthetic_img_path = args.synthetic_images
    synthetic_img_extension = args.synthetic_img_extension
    experiment_folder_name = args.experiment_folder_name

    # Load saved config
    print('Loading configuration...')
    saved_epu_folder = (BASE_PATH / "../results/checkpoints").resolve()
    saved_epu_folder = os.path.join(saved_epu_folder, experiment_folder_name)
    cfg_path = os.path.join(saved_epu_folder, "epu_config.yaml")
    cfg = IUSConfig.from_yaml(cfg_path)

    # Load EPU for obtaining Contribution Feature Profiles of synthetic images
    print('Loading trained model...')
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    trained_ckpt = os.path.join(saved_epu_folder, f"ckpt_{experiment_folder_name}.pt")
    epu = EPUCNNEval(epu_cfg=cfg.model)
    trained_epu = epu.load_ckpt(device=device, ckpt_path=trained_ckpt)

    # Load Baseline Contribution Feature Profile (cb_vector)
    print('IUS class...')
    saved_cb_vec_folder = (BASE_PATH / "../results/cb_vectors").resolve()
    saved_cb_vec_folder = os.path.join(saved_cb_vec_folder, experiment_folder_name)
    cb_vec_path = os.path.join(saved_cb_vec_folder, f"cb_vector_{cb_vec_tag}.npy")
    ius = IUS(cb_path=cb_vec_path, cb_tag=cb_vec_tag, device=device)

    # Load Synthetic Images & Estimate their Contribution Feature Profiles (c^_vectors)
    synthetic_img_path = str((BASE_PATH / ".." / synthetic_img_path).resolve())
    parser = IUSEvalParser(path=synthetic_img_path, image_ext=synthetic_img_extension,)
    synthetic_dset = IUSEvalDataset(dataconfig=cfg.data_params).get_dataset(parser=parser)

    cfg.data_params.data_loading.shuffle = False         # update cfg.data_params.data_loading
    cfg.data_params.data_loading.batch_size = 1
    synthetic_dataloader = to_dataloader(dataset=synthetic_dset, loading_cfg=cfg.data_params.data_loading)

    synthetic_c_vecs = trained_epu.calculate_feature_contribution_profiles(
        data_loader=synthetic_dataloader,
        device=device,
    )

    ius_score_list = ius.ius_measure(c_vectors=synthetic_c_vecs)
    synthetic_filenames = parser.image_filenames

    # Setup Saving Dir
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    output_folder = str((BASE_PATH / "../results/ius_eval").resolve())
    output_folder = os.path.join(output_folder, experiment_folder_name)
    create_output_folders(output_folder)        # skipped if exists

    ius_score_pd = pd.DataFrame(
        {
            "filename": synthetic_filenames,
            "ius_measure_score":  ius_score_list
        }
    )
    file_saved_ius_scores = os.path.join(output_folder, f"ius_scores_{timestamp}.csv")
    ius_score_pd.to_csv(file_saved_ius_scores, index=False)

    save_info_dict = {
        "epu_ckpt": trained_ckpt,
        "config": cfg_path,
        "experiment_id": cfg.experiment_saved_folder_name,
        "cb_vector_path": cb_vec_path,
        "synthetic_images_path": synthetic_img_path,
        "synthetic_img_extension": synthetic_img_extension,
        "synthetic_samples_num": len(synthetic_filenames),
        "timestamp": timestamp,
        "ius_scores": file_saved_ius_scores,
    }
    json_file_path = os.path.join(output_folder, f"ius_scores_info_{timestamp}.json")
    save_to_json(save_info_dict, json_file_path)


if __name__ == "__main__":
    # python - m scripts.eval_ius - -experiment_folder_name ius_dataset_name_base_one_0000_timestamp - -cb_vector_tag normal - -synthetic_images datasets_synthetic/dataset_name/normal - -synthetic_img_extension png
    # python - m scripts.eval_ius - -experiment_folder_name ius_dataset_name_base_one_0000_timestamp - -cb_vector_tag normal - -synthetic_images datasets_synthetic/dataset_name/normal/image_0001.png - -synthetic_img_extension png

    main()