data/augmentation.py

import math
from typing import Tuple, List
import numpy as np
from PIL import Image, ImageOps
from torchvision import transforms
from torchvision.transforms import functional as TF


class Augmentation:
    def __init__(self, p: float) -> None:
        self.p = p

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        raise NotImplementedError

    def __call__(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        if np.random.random_sample() < self.p:
            return self.transform(image, bbox, gaze, head_mask, size)
        return image, bbox, gaze, head_mask, size


class AugmentationList:
    def __init__(self, augmentations: List[Augmentation]) -> None:
        self.augmentations = augmentations

    def __call__(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        for aug in self.augmentations:
            image, bbox, gaze, head_mask, size = aug(image, bbox, gaze, head_mask, size)
        return image, bbox, gaze, head_mask, size


class BoxJitter(Augmentation):
    # Jitter (expansion-only) bounding box size
    def __init__(self, p: float, expansion: float = 0.2) -> None:
        super().__init__(p)
        self.expansion = expansion

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        x_min, y_min, x_max, y_max = bbox
        width, height = size
        k = np.random.random_sample() * self.expansion
        x_min = np.clip(x_min - k * abs(x_max - x_min), 0, width - 1)
        y_min = np.clip(y_min - k * abs(y_max - y_min), 0, height - 1)
        x_max = np.clip(x_max + k * abs(x_max - x_min), 0, width - 1)
        y_max = np.clip(y_max + k * abs(y_max - y_min), 0, height - 1)
        return image, (x_min, y_min, x_max, y_max), gaze, head_mask, size


class RandomCrop(Augmentation):
    def __init__(self, p: float) -> None:
        super().__init__(p)

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        x_min, y_min, x_max, y_max = bbox
        gaze_x, gaze_y = gaze
        width, height = size
        # Calculate the minimum valid range of the crop that doesn't exclude the face and the gaze target
        crop_x_min = np.min([gaze_x * width, x_min, x_max])
        crop_y_min = np.min([gaze_y * height, y_min, y_max])
        crop_x_max = np.max([gaze_x * width, x_min, x_max])
        crop_y_max = np.max([gaze_y * height, y_min, y_max])

        # Randomly select a random top left corner
        crop_x_min = np.random.uniform(0, crop_x_min)
        crop_y_min = np.random.uniform(0, crop_y_min)

        # Find the range of valid crop width and height starting from the (crop_x_min, crop_y_min)
        crop_width_min = crop_x_max - crop_x_min
        crop_height_min = crop_y_max - crop_y_min
        crop_width_max = width - crop_x_min
        crop_height_max = height - crop_y_min

        # Randomly select a width and a height
        crop_width = np.random.uniform(crop_width_min, crop_width_max)
        crop_height = np.random.uniform(crop_height_min, crop_height_max)

        # Round to integers
        crop_y_min, crop_x_min, crop_height, crop_width = map(
            int, map(round, (crop_y_min, crop_x_min, crop_height, crop_width))
        )

        # Crop it
        image = TF.crop(image, crop_y_min, crop_x_min, crop_height, crop_width)
        head_mask = TF.crop(head_mask, crop_y_min, crop_x_min, crop_height, crop_width)

        # convert coordinates into the cropped frame
        x_min, y_min, x_max, y_max = (
            x_min - crop_x_min,
            y_min - crop_y_min,
            x_max - crop_x_min,
            y_max - crop_y_min,
        )

        gaze_x = (gaze_x * width - crop_x_min) / float(crop_width)
        gaze_y = (gaze_y * height - crop_y_min) / float(crop_height)

        return (
            image,
            (x_min, y_min, x_max, y_max),
            (gaze_x, gaze_y),
            head_mask,
            (crop_width, crop_height),
        )


class RandomFlip(Augmentation):
    def __init__(self, p: float) -> None:
        super().__init__(p)

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        image = image.transpose(Image.FLIP_LEFT_RIGHT)
        head_mask = head_mask.transpose(Image.FLIP_LEFT_RIGHT)
        x_min, y_min, x_max, y_max = bbox
        x_min, x_max = size[0] - x_max, size[0] - x_min
        gaze_x, gaze_y = 1 - gaze[0], gaze[1]
        return image, (x_min, y_min, x_max, y_max), (gaze_x, gaze_y), head_mask, size


class RandomRotate(Augmentation):
    def __init__(
        self, p: float, max_angle: int = 20, resample: int = Image.BILINEAR
    ) -> None:
        super().__init__(p)
        self.max_angle = max_angle
        self.resample = resample

    def _random_rotation_matrix(self):
        angle = (2 * np.random.random_sample() - 1) * self.max_angle
        angle = -math.radians(angle)
        return [
            round(math.cos(angle), 15),
            round(math.sin(angle), 15),
            0.0,
            round(-math.sin(angle), 15),
            round(math.cos(angle), 15),
            0.0,
        ]

    @staticmethod
    def _transform(x, y, matrix):
        return (
            matrix[0] * x + matrix[1] * y + matrix[2],
            matrix[3] * x + matrix[4] * y + matrix[5],
        )

    @staticmethod
    def _inv_transform(x, y, matrix):
        x, y = x - matrix[2], y - matrix[5]
        return matrix[0] * x + matrix[3] * y, matrix[1] * x + matrix[4] * y

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        x_min, y_min, x_max, y_max = bbox
        gaze_x, gaze_y = gaze
        width, height = size
        rot_mat = self._random_rotation_matrix()

        # Calculate offsets
        rot_center = (width / 2.0, height / 2.0)
        rot_mat[2], rot_mat[5] = self._transform(
            -rot_center[0], -rot_center[1], rot_mat
        )
        rot_mat[2] += rot_center[0]
        rot_mat[5] += rot_center[1]
        xx = []
        yy = []
        for x, y in ((0, 0), (width, 0), (width, height), (0, height)):
            x, y = self._transform(x, y, rot_mat)
            xx.append(x)
            yy.append(y)
        nw = math.ceil(max(xx)) - math.floor(min(xx))
        nh = math.ceil(max(yy)) - math.floor(min(yy))
        rot_mat[2], rot_mat[5] = self._transform(
            -(nw - width) / 2.0, -(nh - height) / 2.0, rot_mat
        )

        image = image.transform((nw, nh), Image.AFFINE, rot_mat, self.resample)
        head_mask = head_mask.transform((nw, nh), Image.AFFINE, rot_mat, self.resample)

        xx = []
        yy = []
        for x, y in (
            (x_min, y_min),
            (x_min, y_max),
            (x_max, y_min),
            (x_max, y_max),
        ):
            x, y = self._inv_transform(x, y, rot_mat)
            xx.append(x)
            yy.append(y)
        x_max, x_min = min(max(xx), nw), max(min(xx), 0)
        y_max, y_min = min(max(yy), nh), max(min(yy), 0)

        gaze_x, gaze_y = self._inv_transform(gaze_x * width, gaze_y * height, rot_mat)
        gaze_x = max(min(gaze_x / nw, 1), 0)
        gaze_y = max(min(gaze_y / nh, 1), 0)

        return (
            image,
            (x_min, y_min, x_max, y_max),
            (gaze_x, gaze_y),
            head_mask,
            (nw, nh),
        )


class ColorJitter(Augmentation):
    def __init__(
        self,
        p: float,
        brightness: float = 0.4,
        contrast: float = 0.4,
        saturation: float = 0.2,
        hue: float = 0.1,
    ) -> None:
        super().__init__(p)
        self.color_jitter = transforms.ColorJitter(
            brightness=brightness, contrast=contrast, saturation=saturation, hue=hue
        )

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        return self.color_jitter(image), bbox, gaze, head_mask, size


class RandomLSJ(Augmentation):
    def __init__(self, p: float, min_scale: float = 0.1) -> None:
        super().__init__(p)
        self.min_scale = min_scale

    def transform(
        self,
        image: Image,
        bbox: Tuple[float],
        gaze: Tuple[float],
        head_mask: Image,
        size: Tuple[int],
    ):
        x_min, y_min, x_max, y_max = bbox
        gaze_x, gaze_y = gaze
        width, height = size

        scale = self.min_scale + np.random.random_sample() * (1 - self.min_scale)
        nh, nw = int(height * scale), int(width * scale)

        image = TF.resize(image, (nh, nw))
        image = ImageOps.expand(image, (0, 0, width - nw, height - nh))
        head_mask = TF.resize(head_mask, (nh, nw))
        head_mask = ImageOps.expand(head_mask, (0, 0, width - nw, height - nh))

        x_min, y_min, x_max, y_max = (
            x_min * scale,
            y_min * scale,
            x_max * scale,
            y_max * scale,
        )
        gaze_x, gaze_y = gaze_x * scale, gaze_y * scale
        return image, (x_min, y_min, x_max, y_max), (gaze_x, gaze_y), head_mask, size