File size: 3,240 Bytes
c7c3133
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""Optional (--from-photo): the structure maps of the hed and depth tasks, computed from a photo.

    hed     soft edges of ControlNet's HED annotator, white on black
    depth   relative inverse depth of Depth Anything V2 Small, bright = near

Both networks are downloaded from the Hugging Face Hub under their own licenses on first use.
"""
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image


class _HEDBlock(torch.nn.Module):
    def __init__(self, cin, cout, n):
        super().__init__()
        self.convs = torch.nn.Sequential(*([torch.nn.Conv2d(cin, cout, 3, 1, 1)] + [torch.nn.Conv2d(cout, cout, 3, 1, 1) for _ in range(n - 1)]))
        self.projection = torch.nn.Conv2d(cout, 1, 1, 1, 0)

    def forward(self, x, down=False):
        h = F.max_pool2d(x, 2, 2) if down else x
        for c in self.convs:
            h = F.relu(c(h))
        return h, self.projection(h)


class _ControlNetHED(torch.nn.Module):
    """the HED network of ControlNet 1.1's annotator: five conv stages, each with a 1 x 1 side output"""

    def __init__(self):
        super().__init__()
        self.norm = torch.nn.Parameter(torch.zeros(1, 3, 1, 1))
        self.block1, self.block2, self.block3 = _HEDBlock(3, 64, 2), _HEDBlock(64, 128, 2), _HEDBlock(128, 256, 3)
        self.block4, self.block5 = _HEDBlock(256, 512, 3), _HEDBlock(512, 512, 3)

    def forward(self, x):
        h, ps = x - self.norm, []
        for k, b in enumerate((self.block1, self.block2, self.block3, self.block4, self.block5)):
            h, p = b(h, down=k > 0)
            ps.append(p)
        return ps


@torch.no_grad()
def hed_map(img, device="cuda"):
    """uint8 [H, W, 3] photo -> uint8 [H, W, 3] soft edge map (mean of the five side outputs, sigmoid)"""
    from huggingface_hub import hf_hub_download
    net = _ControlNetHED()
    net.load_state_dict(torch.load(hf_hub_download("lllyasviel/Annotators", "ControlNetHED.pth"), map_location="cpu"), strict=True)
    net = net.to(device).eval()
    h, w = img.shape[:2]
    ps = net(torch.from_numpy(img.astype(np.float32)).permute(2, 0, 1)[None].to(device))
    m = torch.stack([F.interpolate(p, size=(h, w), mode="bilinear", align_corners=False)[0, 0] for p in ps]).mean(0)
    e = torch.sigmoid(m).cpu().numpy()
    return np.repeat((e * 255).round().astype(np.uint8)[..., None], 3, axis=2)


@torch.no_grad()
def depth_map(img, device="cuda"):
    """uint8 [H, W, 3] photo -> uint8 [H, W, 3] depth map, bright = near"""
    from transformers import AutoImageProcessor, AutoModelForDepthEstimation
    name = "depth-anything/Depth-Anything-V2-Small-hf"
    proc = AutoImageProcessor.from_pretrained(name)
    net = AutoModelForDepthEstimation.from_pretrained(name).to(device).eval()
    o = net(**{k: v.to(device) for k, v in proc(images=Image.fromarray(img), return_tensors="pt").items()}).predicted_depth
    d = F.interpolate(o[None], size=img.shape[:2], mode="bicubic", align_corners=False)[0, 0].cpu().numpy()
    d = (d - d.min()) / max(float(d.max() - d.min()), 1e-6)
    return np.repeat((d * 255).round().astype(np.uint8)[..., None], 3, axis=2)


def structure_map(img, task, device="cuda"):
    return {"hed": hed_map, "depth": depth_map}[task](img, device)