python/utils/vision_output.py

from typing import List, Optional, Sequence, Tuple

import numpy as np


def seq_len_from_output(output: np.ndarray) -> Optional[int]:
    if output.ndim < 2:
        return None
    if output.ndim == 2:
        return int(output.shape[0])
    return int(output.shape[-2])


def normalize_vit_output(
    output: np.ndarray,
    target_hidden_size: int,
    expected_tokens: Optional[int] = None,
) -> np.ndarray:
    normalized = output
    if expected_tokens is not None:
        if normalized.ndim == 3 and normalized.shape[1] == target_hidden_size and normalized.shape[2] == expected_tokens:
            normalized = np.transpose(normalized, (0, 2, 1))
        elif normalized.ndim == 2 and normalized.shape[0] == target_hidden_size and normalized.shape[1] == expected_tokens:
            normalized = np.transpose(normalized, (1, 0))
    return normalized


def describe_output_shapes(outputs: Sequence[np.ndarray]) -> List[Tuple[int, ...]]:
    return [tuple(int(v) for v in output.shape) for output in outputs]


def select_vit_output(
    outputs: Sequence[np.ndarray],
    target_hidden_size: int,
    expected_tokens: Optional[int] = None,
) -> np.ndarray:
    normalized_outputs = [
        normalize_vit_output(output, target_hidden_size, expected_tokens=expected_tokens) for output in outputs
    ]

    image_embeds = None
    if expected_tokens is not None:
        for output in normalized_outputs:
            if output.ndim >= 2 and seq_len_from_output(output) == expected_tokens and output.shape[-1] == target_hidden_size:
                image_embeds = output
                break
        if image_embeds is None:
            for output in normalized_outputs:
                if output.ndim >= 2 and seq_len_from_output(output) == expected_tokens:
                    image_embeds = output
                    break

    if image_embeds is None:
        for output in normalized_outputs:
            if output.ndim >= 2 and output.shape[-1] == target_hidden_size:
                image_embeds = output
                break

    if image_embeds is None:
        image_embeds = normalized_outputs[0]
    if image_embeds.ndim == 2:
        image_embeds = image_embeds[None, ...]
    return image_embeds