viz_utils.py

import json
import math
from datetime import date

import numpy as np
from matplotlib import colormaps
from PIL import ImageColor, ImageDraw, ImageFont

today = date.today()
FONTS = {
    "amiko": "fonts/Amiko-Regular.ttf",
    "nature": "fonts/LoveNature.otf",
    "painter": "fonts/PainterDecorator.otf",
    "animals": "fonts/UncialAnimals.ttf",
    "zen": "fonts/ZEN.TTF",
}
# perceptually uniform maps first; turbo separates neighbouring bodyparts best
COLORMAPS = ["viridis", "plasma", "magma", "cividis", "turbo"]


#########################################
#  Draw keypoints on image
def draw_keypoints_on_image(
    image,
    keypoints,
    map_label_id_to_str,
    flag_show_str_labels,
    use_normalized_coordinates=True,
    font_style="amiko",
    font_size=8,
    keypt_color="#ff0000",
    marker_size=2,
    color_by_confidence=True,
    colormap="viridis",
):
    """Draws keypoints on an image.
    Modified from:
        https://www.programcreek.com/python/?code=fjchange%2Fobject_centric_VAD%2Fobject_centric_VAD-master%2Fobject_detection%2Futils%2Fvisualization_utils.py
    Args:
    image: a PIL.Image object.
    keypoints: a numpy array with shape [num_keypoints, 2].
    map_label_id_to_str: dict with keys=label number and values= label string
    flag_show_str_labels: boolean to select whether or not to show string labels
    color: color to draw the keypoints with. Default is red.
    radius: keypoint radius. Default value is 2.
    use_normalized_coordinates: if True (default), treat keypoint values as
        relative to the image.  Otherwise treat them as absolute.


    """
    # get a drawing context
    draw = ImageDraw.Draw(image, "RGBA")

    im_width, im_height = image.size
    keypoints_x = [k[0] for k in keypoints]
    keypoints_y = [k[1] for k in keypoints]
    confidences = [k[2] for k in keypoints]

    # adjust keypoints coords if required
    if use_normalized_coordinates:
        keypoints_x = tuple([im_width * x for x in keypoints_x])
        keypoints_y = tuple([im_height * y for y in keypoints_y])

    cmap = colormaps[colormap]
    # draw ellipses around keypoints
    for i, (keypoint_x, keypoint_y) in enumerate(zip(keypoints_x, keypoints_y, strict=True)):
        # handling potential nans in the keypoints
        if np.isnan(keypoint_x).any():
            continue

        confidence = float(np.clip(confidences[i], 0, 1))
        if color_by_confidence:
            # fill color encodes the keypoint confidence (see confidence_legend_html in ui_utils)
            round_fill = cmap(confidence, bytes=True)
        else:
            # one color per bodypart, transparency encodes the confidence
            round_fill = list(cmap(i / max(len(keypoints) - 1, 1), bytes=True))
            round_fill[3] = round(confidence * 255)
            round_fill = tuple(round_fill)
        draw.ellipse(
            [
                (keypoint_x - marker_size, keypoint_y - marker_size),
                (keypoint_x + marker_size, keypoint_y + marker_size),
            ],
            fill=tuple(round_fill),
            outline="black",
            width=1,
        )  # fill and outline: [0,255]

        # add string labels around keypoints
        if flag_show_str_labels:
            font = ImageFont.truetype(FONTS[font_style], font_size)
            draw.text(
                (keypoint_x + marker_size, keypoint_y + marker_size),  # (0.5*im_width, 0.5*im_height), #-------
                display_bodypart(map_label_id_to_str[i]),
                ImageColor.getcolor(keypt_color, "RGB"),  # rgb #
                font=font,
            )


#########################################
#  Bodypart names for display
# display names where the SuperAnimal definitions misspell (quadruped "thai") or read oddly
# (top-view mouse "backend"); the JSON output keeps the model's names
BODYPART_DISPLAY_NAMES = {
    "front_left_thai": "front left thigh",
    "front_right_thai": "front right thigh",
    "back_left_thai": "back left thigh",
    "back_right_thai": "back right thigh",
    "mid_backend": "mid back end",
    "mid_backend2": "mid back end 2",
    "mid_backend3": "mid back end 3",
}


def display_bodypart(name):
    return BODYPART_DISPLAY_NAMES.get(name, name.replace("_", " "))


#########################################
#  Keypoint confidences as table rows
def keypoint_confidence_rows(kpts_per_animal, map_label_id_to_str):
    """(animal, bodypart, confidence) for every keypoint kept (not NaN), lowest confidence first."""
    rows = []
    for i_animal, kpts in enumerate(kpts_per_animal):
        for i_kpt, kpt in enumerate(kpts):
            if not np.isnan(kpt[2]):
                rows.append([i_animal, display_bodypart(map_label_id_to_str[i_kpt]), round(float(kpt[2]), 3)])
    return sorted(rows, key=lambda row: row[2])


#########################################
#  Save the annotated image for download
def save_annotated_image(image, path_to_output_file="download_annotated.png"):
    image.save(path_to_output_file)
    return path_to_output_file


#########################################
#  Draw bboxes on image
def draw_bbox_w_text(img, results, font_style="amiko", font_size=8, bbox_color="#ff0000"):
    x1, y1, x2, y2, confidence = results[:5]
    draw = ImageDraw.Draw(img)
    draw.rectangle([(x1, y1), (x2, y2)], outline=bbox_color, width=max(2, round(font_size / 5)))

    label = f"animal {confidence:.2f}"
    font = ImageFont.truetype(FONTS[font_style], font_size)
    left, top, right, bottom = draw.textbbox((0, 0), label, font=font)
    pad = max(2, font_size // 5)
    label_w, label_h = right - left + 2 * pad, bottom - top + 2 * pad
    # label above the box, or inside it when the box touches the top of the image
    label_y = y1 - label_h if y1 >= label_h else y1
    draw.rectangle([(x1, label_y), (x1 + label_w, label_y + label_h)], fill=bbox_color)
    draw.text((x1 + pad - left, label_y + pad - top), label, font=font, fill=label_text_color(bbox_color))


def label_text_color(background):
    # black or white, whichever contrasts more with the background (WCAG relative luminance)
    channels = [c / 255 for c in ImageColor.getrgb(background)[:3]]
    r, g, b = [c / 12.92 if c <= 0.03928 else ((c + 0.055) / 1.055) ** 2.4 for c in channels]
    return "black" if 0.2126 * r + 0.7152 * g + 0.0722 * b > 0.179 else "white"


###########################################
#  JSON outputs: pixel coordinates in the input image, hidden keypoints as null
COORDINATES = "pixels in the input image (after EXIF orientation), origin top-left, x right, y down"


def keypoints_to_json(kpts, offset=(0, 0), scale=(1.0, 1.0)):
    """[x, y, confidence] per keypoint, mapped by (k + offset) * scale; NaN (below threshold) becomes null."""
    out = []
    for x, y, conf in np.asarray(kpts, dtype=float)[:, :3]:
        if np.isnan(conf):
            out.append(None)
        else:
            out.append([(x + offset[0]) * scale[0], (y + offset[1]) * scale[1], conf])
    return out


def write_json(info, path_to_output_file):
    with open(path_to_output_file, "w") as f:
        json.dump(info, f, indent=1, allow_nan=False)
    return path_to_output_file


def json_header(image_size, annotated_size):
    return {
        "date": str(today),
        "coordinates": COORDINATES,
        "image_size": list(image_size),
        "annotated_image_size": list(annotated_size),
    }


def save_results_as_json(
    md_results,
    dlc_outputs,
    animal_bboxes,
    map_dlc_label_id_to_str,
    model,
    mega_model_input,
    image_size,
    path_to_output_file="download_predictions.json",
):
    """TF (legacy) MegaDetector + DLC results.

    animal_bboxes: detection rows [x1,y1,x2,y2,conf,label] in the MegaDetector frame, one per entry of dlc_outputs
    dlc_outputs: keypoints relative to each crop (crops start at floor(x1), floor(y1), see crop_animal_detections)
    """
    md_h, md_w = md_results.ims[0].shape[:2]
    scale = (image_size[0] / md_w, image_size[1] / md_h)
    info = json_header(image_size, (md_w, md_h))
    info["MD_model"] = str(mega_model_input)
    info["number_of_bb"] = len(dlc_outputs)
    info["dlc_model"] = model
    labels = list(map_dlc_label_id_to_str.values())

    for i, kpts in enumerate(dlc_outputs):
        x1, y1, x2, y2, confidence, _ = animal_bboxes[i]
        info["bb_" + str(i)] = {
            "corner_1": (x1 * scale[0], y1 * scale[1]),
            "corner_2": (x2 * scale[0], y2 * scale[1]),
            "predict MD": md_results.names[0],
            "confidence MD": float(confidence),
            "dlc_pred": dict(
                zip(labels, keypoints_to_json(kpts, offset=(math.floor(x1), math.floor(y1)), scale=scale), strict=True)
            ),
        }
    return write_json(info, path_to_output_file)


def save_results_only_dlc(
    dlc_outputs, map_label_id_to_str, model, image_size, output_file="dowload_predictions_dlc.json"
):
    """TF (legacy) DLC run on the whole input image (keypoints already in input pixels)."""
    info = json_header(image_size, image_size)
    info["dlc_model"] = model
    info["dlc_pred"] = dict(zip(map_label_id_to_str.values(), keypoints_to_json(dlc_outputs), strict=True))
    return write_json(info, output_file)


def save_results_pytorch(
    animals,
    map_label_id_to_str,
    model,
    pose_model,
    detector,
    image_size,
    annotated_size,
    path_to_output_file="download_predictions.json",
):
    """PyTorch SuperAnimal results (same layout as save_results_as_json).

    animals: list of {'bbox': [x1,y1,x2,y2,conf], 'kpts': (num_keypoints, 3)}, in the annotated (resized) image
    detector: None if the detector was skipped (whole image used as one animal)
    """
    scale = (image_size[0] / annotated_size[0], image_size[1] / annotated_size[1])
    info = json_header(image_size, annotated_size)
    info["backend"] = "pytorch"
    info["dlc_model"] = model
    info["pose_model"] = pose_model
    info["detector"] = detector
    info["number_of_bb"] = len(animals)
    labels = list(map_label_id_to_str.values())

    for i, animal in enumerate(animals):
        x1, y1, x2, y2, confidence = animal["bbox"]
        info["bb_" + str(i)] = {
            "corner_1": (x1 * scale[0], y1 * scale[1]),
            "corner_2": (x2 * scale[0], y2 * scale[1]),
            "confidence": confidence,
            "dlc_pred": dict(zip(labels, keypoints_to_json(animal["kpts"], scale=scale), strict=True)),
        }
    return write_json(info, path_to_output_file)


###########################################