Spaces:
Running on Zero
Running on Zero
File size: 3,240 Bytes
c7c3133 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """Optional (--from-photo): the structure maps of the hed and depth tasks, computed from a photo.
hed soft edges of ControlNet's HED annotator, white on black
depth relative inverse depth of Depth Anything V2 Small, bright = near
Both networks are downloaded from the Hugging Face Hub under their own licenses on first use.
"""
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
class _HEDBlock(torch.nn.Module):
def __init__(self, cin, cout, n):
super().__init__()
self.convs = torch.nn.Sequential(*([torch.nn.Conv2d(cin, cout, 3, 1, 1)] + [torch.nn.Conv2d(cout, cout, 3, 1, 1) for _ in range(n - 1)]))
self.projection = torch.nn.Conv2d(cout, 1, 1, 1, 0)
def forward(self, x, down=False):
h = F.max_pool2d(x, 2, 2) if down else x
for c in self.convs:
h = F.relu(c(h))
return h, self.projection(h)
class _ControlNetHED(torch.nn.Module):
"""the HED network of ControlNet 1.1's annotator: five conv stages, each with a 1 x 1 side output"""
def __init__(self):
super().__init__()
self.norm = torch.nn.Parameter(torch.zeros(1, 3, 1, 1))
self.block1, self.block2, self.block3 = _HEDBlock(3, 64, 2), _HEDBlock(64, 128, 2), _HEDBlock(128, 256, 3)
self.block4, self.block5 = _HEDBlock(256, 512, 3), _HEDBlock(512, 512, 3)
def forward(self, x):
h, ps = x - self.norm, []
for k, b in enumerate((self.block1, self.block2, self.block3, self.block4, self.block5)):
h, p = b(h, down=k > 0)
ps.append(p)
return ps
@torch.no_grad()
def hed_map(img, device="cuda"):
"""uint8 [H, W, 3] photo -> uint8 [H, W, 3] soft edge map (mean of the five side outputs, sigmoid)"""
from huggingface_hub import hf_hub_download
net = _ControlNetHED()
net.load_state_dict(torch.load(hf_hub_download("lllyasviel/Annotators", "ControlNetHED.pth"), map_location="cpu"), strict=True)
net = net.to(device).eval()
h, w = img.shape[:2]
ps = net(torch.from_numpy(img.astype(np.float32)).permute(2, 0, 1)[None].to(device))
m = torch.stack([F.interpolate(p, size=(h, w), mode="bilinear", align_corners=False)[0, 0] for p in ps]).mean(0)
e = torch.sigmoid(m).cpu().numpy()
return np.repeat((e * 255).round().astype(np.uint8)[..., None], 3, axis=2)
@torch.no_grad()
def depth_map(img, device="cuda"):
"""uint8 [H, W, 3] photo -> uint8 [H, W, 3] depth map, bright = near"""
from transformers import AutoImageProcessor, AutoModelForDepthEstimation
name = "depth-anything/Depth-Anything-V2-Small-hf"
proc = AutoImageProcessor.from_pretrained(name)
net = AutoModelForDepthEstimation.from_pretrained(name).to(device).eval()
o = net(**{k: v.to(device) for k, v in proc(images=Image.fromarray(img), return_tensors="pt").items()}).predicted_depth
d = F.interpolate(o[None], size=img.shape[:2], mode="bicubic", align_corners=False)[0, 0].cpu().numpy()
d = (d - d.min()) / max(float(d.max() - d.min()), 1e-6)
return np.repeat((d * 255).round().astype(np.uint8)[..., None], 3, axis=2)
def structure_map(img, task, device="cuda"):
return {"hed": hed_map, "depth": depth_map}[task](img, device)
|