OpenOOD/openood/postprocessors/patchcore_postprocessor.py

from __future__ import absolute_import, division, print_function

import abc
import os

import faiss
import numpy as np
import torch
from sklearn.metrics import pairwise_distances
from sklearn.random_projection import SparseRandomProjection
from torch import nn
from torch.nn import functional as F
from tqdm import tqdm

from .base_postprocessor import BasePostprocessor


def embedding_concat(x, y):
    B, C1, H1, W1 = x.size()
    _, C2, H2, W2 = y.size()
    s = int(H1 / H2)
    x = F.unfold(x, kernel_size=s, dilation=1, stride=s)
    x = x.view(B, C1, -1, H2, W2)
    z = torch.zeros(B, C1 + C2, x.size(2), H2, W2)
    for i in range(x.size(2)):
        z[:, :, i, :, :] = torch.cat((x[:, :, i, :, :], y), 1)
    z = z.view(B, -1, H2 * W2)
    z = F.fold(z, kernel_size=s, output_size=(H1, W1), stride=s)
    return z


def reshape_embedding(embedding):
    embedding_list = []
    for k in range(embedding.shape[0]):
        for i in range(embedding.shape[2]):
            for j in range(embedding.shape[3]):
                embedding_list.append(embedding[k, :, i, j])
    return embedding_list


class PatchcorePostprocessor(BasePostprocessor):
    def __init__(self, config):
        super(PatchcorePostprocessor, self).__init__(config)
        self.config = config
        self.postprocessor_args = config.postprocessor.postprocessor_args
        self.n_neighbors = config.postprocessor.postprocessor_args.n_neighbors
        self.feature_mean, self.feature_prec = None, None
        self.alpha_list = None
        self.gt_list_px_lvl = []
        self.pred_list_px_lvl = []
        self.gt_list_img_lvl = []
        self.pred_list_img_lvl = []
        self.img_path_list = []
        self.features = []

    def setup(self, net: nn.Module, id_loader_dict, ood_loader_dict):
        # step 1:
        self.model = net
        # on train start
        self.model.eval()  # to stop running_var move (maybe not critical)
        self.embedding_list = []

        if (self.config.network.load_cached_faiss):
            path = self.config.output_dir
            # load index
            if os.path.isfile(os.path.join(path, 'index.faiss')):
                self.index = faiss.read_index(os.path.join(
                    path, 'index.faiss'))
                if torch.cuda.is_available():
                    res = faiss.StandardGpuResources()
                    self.index = faiss.index_cpu_to_gpu(res, 0, self.index)
                self.init_results_list()
                return

        # training step
        train_dataiter = iter(id_loader_dict['train'])

        for train_step in tqdm(range(1,
                                     len(train_dataiter) + 1),
                               position=0,
                               leave=True):
            batch = next(train_dataiter)
            x = batch['data'].cuda()
            features = self.model.forward(x, return_feature=True)
            embeddings = []
            for feature in features:
                m = torch.nn.AvgPool2d(9, 1, 1)
                embeddings.append(m(feature))
            embedding = embedding_concat(embeddings[0], embeddings[1])
            self.embedding_list.extend(reshape_embedding(np.array(embedding)))

        # training end
        total_embeddings = np.array(self.embedding_list)

        # Random projection
        print('Random projection')
        self.randomprojector = SparseRandomProjection(
            n_components='auto',
            eps=0.9)  # 'auto' => Johnson-Lindenstrauss lemma
        self.randomprojector.fit(total_embeddings)
        # Coreset Subsampling
        print('Coreset Subsampling')
        selector = kCenterGreedy(total_embeddings, 0, 0)
        selected_idx = selector.select_batch(
            model=self.randomprojector,
            already_selected=[],
            N=int(total_embeddings.shape[0] *
                  self.postprocessor_args.coreset_sampling_ratio))
        self.embedding_coreset = total_embeddings[selected_idx]

        print('initial embedding size : ', total_embeddings.shape)
        print('final embedding size : ', self.embedding_coreset.shape)
        # faiss
        print('faiss indexing')
        self.index = faiss.IndexFlatL2(self.embedding_coreset.shape[1])
        self.index.add(self.embedding_coreset)
        if not os.path.isdir(os.path.join('./results/patch/')):
            os.mkdir('./results/patch/')
        faiss.write_index(self.index,
                          os.path.join('./results/patch/', 'index.faiss'))

    def init_results_list(self):
        self.gt_list_px_lvl = []
        self.pred_list_px_lvl = []
        self.gt_list_img_lvl = []
        self.pred_list_img_lvl = []

    def postprocess(self, net: nn.Module, data):

        self.init_results_list()
        score_patch = []
        # extract embedding
        for x in data.split(1, dim=0):
            features = self.model.forward(x, return_feature=True)
            embeddings = []
            for feature in features:
                m = torch.nn.AvgPool2d(3, 1, 1)
                embeddings.append(m(feature))
            embedding_ = embedding_concat(embeddings[0], embeddings[1])
            embedding_test = np.array(reshape_embedding(np.array(embedding_)))
            score_patches, _ = self.index.search(embedding_test,
                                                 k=self.n_neighbors)

            score_patch.append(score_patches)

            N_b = score_patches[np.argmax(score_patches[:, 0])]
            w = (1 - (np.max(np.exp(N_b)) / np.sum(np.exp(N_b))))
            score = w * max(score_patches[:, 0])  # Image-level score

            self.pred_list_img_lvl.append(score)

        pred = []
        for i in self.pred_list_img_lvl:
            # 6.3 is the trial value.
            if (i > 6.3):
                pred.append(torch.tensor(1))
            else:
                pred.append(torch.tensor(-1))
        conf = []
        for i in score_patch:
            conf.append(i)
        conf = torch.tensor(conf, dtype=torch.float32)
        conf = conf.cuda()

        pred_list_img_lvl = []

        for patchscore in np.concatenate([conf.cpu().tolist()]):
            N_b = patchscore[np.argmax(patchscore[:, 0])]
            w = (1 - (np.max(np.exp(N_b)) / np.sum(np.exp(N_b))))
            score = w * max(patchscore[:, 0])  # Image-level score

            pred_list_img_lvl.append(score)

        if self.config.evaluator.name == 'patch':
            return pred, conf
        else:
            return pred, -1 * torch.tensor(pred_list_img_lvl).cuda()


# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Abstract class for sampling methods.

Provides interface to sampling methods that allow same signature for
select_batch.  Each subclass implements select_batch_ with the desired
signature for readability.
"""


class SamplingMethod(object):
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def __init__(self, X, y, seed, **kwargs):
        self.X = X
        self.y = y
        self.seed = seed

    def flatten_X(self):
        shape = self.X.shape
        flat_X = self.X
        if len(shape) > 2:
            flat_X = np.reshape(self.X, (shape[0], np.product(shape[1:])))
        return flat_X

    @abc.abstractmethod
    def select_batch_(self):
        return

    def select_batch(self, **kwargs):
        return self.select_batch_(**kwargs)

    def to_dict(self):
        return None


# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Returns points that minimizes the maximum distance of any point to a center.

Implements the k-Center-Greedy method in
Ozan Sener and Silvio Savarese.  A Geometric Approach to Active Learning for
Convolutional Neural Networks. https://arxiv.org/abs/1708.00489 2017

Distance metric defaults to l2 distance.  Features used to calculate distance
are either raw features or if a model has transform method then uses the output
of model.transform(X).

Can be extended to a robust k centers algorithm that ignores a certain number
of outlier datapoints.
Resulting centers are solution to multiple integer program.
"""


class kCenterGreedy(SamplingMethod):
    def __init__(self, X, y, seed, metric='euclidean'):
        self.X = X
        self.y = y
        self.flat_X = self.flatten_X()
        self.name = 'kcenter'
        self.features = self.flat_X
        self.metric = metric
        self.min_distances = None
        self.n_obs = self.X.shape[0]
        self.already_selected = []

    def update_distances(self,
                         cluster_centers,
                         only_new=True,
                         reset_dist=False):
        """Update min distances given cluster centers.

        Args:
          cluster_centers: indices of cluster centers
          only_new: only calculate distance for newly selected points and
          update min_distances.
          rest_dist: whether to reset min_distances.
        """

        if reset_dist:
            self.min_distances = None
        if only_new:
            cluster_centers = [
                d for d in cluster_centers if d not in self.already_selected
            ]
        if cluster_centers:
            # Update min_distances for all examples given new cluster center.
            x = self.features[cluster_centers]
            dist = pairwise_distances(self.features, x, metric=self.metric)

            if self.min_distances is None:
                self.min_distances = np.min(dist, axis=1).reshape(-1, 1)
            else:
                self.min_distances = np.minimum(self.min_distances, dist)

    def select_batch_(self, model, already_selected, N, **kwargs):
        """Diversity promoting active learning method that greedily forms a
        batch to minimize the maximum distance to a cluster center among all
        unlabeled datapoints.

        Args:
          model: model with scikit-like API with decision_function implemented
          already_selected: index of datapoints already selected
          N: batch size

        Returns:
          indices of points selected to minimize distance to cluster centers
        """

        try:
            # Assumes that the transform function takes in original data and
            # not flattened data.
            print('Getting transformed features...')
            self.features = model.transform(self.X)
            print('Calculating distances...')
            self.update_distances(already_selected,
                                  only_new=False,
                                  reset_dist=True)
        except:
            print('Using flat_X as features.')
            self.update_distances(already_selected,
                                  only_new=True,
                                  reset_dist=False)

        new_batch = []

        for _ in tqdm(range(N)):
            if self.already_selected is None:
                # Initialize centers with a randomly selected datapoint
                ind = np.random.choice(np.arange(self.n_obs))
            else:
                ind = np.argmax(self.min_distances)
            # New examples should not be in already selected since those points
            # should have min_distance of zero to a cluster center.
            assert ind not in already_selected

            self.update_distances([ind], only_new=True, reset_dist=False)
            new_batch.append(ind)
        print('Maximum distance from cluster centers is %0.2f' %
              max(self.min_distances))

        self.already_selected = already_selected

        return new_batch