Spaces:
Running on Zero
Running on Zero
Download jeveration/annotators.py from linlinlinlinsdada/Jeveration: direct link, hf CLI and curl.
- Browser
- Download file 3.24 kB
-
https://huggingface.co/spaces/linlinlinlinsdada/Jeveration/resolve/main/jeveration/annotators.py
- Command line
-
hf download hf://spaces/linlinlinlinsdada/Jeveration/jeveration/annotators.py
-
curl -L -o annotators.py https://huggingface.co/spaces/linlinlinlinsdada/Jeveration/resolve/main/jeveration/annotators.py
3.24 kB
| """Optional (--from-photo): the structure maps of the hed and depth tasks, computed from a photo. | |
| hed soft edges of ControlNet's HED annotator, white on black | |
| depth relative inverse depth of Depth Anything V2 Small, bright = near | |
| Both networks are downloaded from the Hugging Face Hub under their own licenses on first use. | |
| """ | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| class _HEDBlock(torch.nn.Module): | |
| def __init__(self, cin, cout, n): | |
| super().__init__() | |
| self.convs = torch.nn.Sequential(*([torch.nn.Conv2d(cin, cout, 3, 1, 1)] + [torch.nn.Conv2d(cout, cout, 3, 1, 1) for _ in range(n - 1)])) | |
| self.projection = torch.nn.Conv2d(cout, 1, 1, 1, 0) | |
| def forward(self, x, down=False): | |
| h = F.max_pool2d(x, 2, 2) if down else x | |
| for c in self.convs: | |
| h = F.relu(c(h)) | |
| return h, self.projection(h) | |
| class _ControlNetHED(torch.nn.Module): | |
| """the HED network of ControlNet 1.1's annotator: five conv stages, each with a 1 x 1 side output""" | |
| def __init__(self): | |
| super().__init__() | |
| self.norm = torch.nn.Parameter(torch.zeros(1, 3, 1, 1)) | |
| self.block1, self.block2, self.block3 = _HEDBlock(3, 64, 2), _HEDBlock(64, 128, 2), _HEDBlock(128, 256, 3) | |
| self.block4, self.block5 = _HEDBlock(256, 512, 3), _HEDBlock(512, 512, 3) | |
| def forward(self, x): | |
| h, ps = x - self.norm, [] | |
| for k, b in enumerate((self.block1, self.block2, self.block3, self.block4, self.block5)): | |
| h, p = b(h, down=k > 0) | |
| ps.append(p) | |
| return ps | |
| def hed_map(img, device="cuda"): | |
| """uint8 [H, W, 3] photo -> uint8 [H, W, 3] soft edge map (mean of the five side outputs, sigmoid)""" | |
| from huggingface_hub import hf_hub_download | |
| net = _ControlNetHED() | |
| net.load_state_dict(torch.load(hf_hub_download("lllyasviel/Annotators", "ControlNetHED.pth"), map_location="cpu"), strict=True) | |
| net = net.to(device).eval() | |
| h, w = img.shape[:2] | |
| ps = net(torch.from_numpy(img.astype(np.float32)).permute(2, 0, 1)[None].to(device)) | |
| m = torch.stack([F.interpolate(p, size=(h, w), mode="bilinear", align_corners=False)[0, 0] for p in ps]).mean(0) | |
| e = torch.sigmoid(m).cpu().numpy() | |
| return np.repeat((e * 255).round().astype(np.uint8)[..., None], 3, axis=2) | |
| def depth_map(img, device="cuda"): | |
| """uint8 [H, W, 3] photo -> uint8 [H, W, 3] depth map, bright = near""" | |
| from transformers import AutoImageProcessor, AutoModelForDepthEstimation | |
| name = "depth-anything/Depth-Anything-V2-Small-hf" | |
| proc = AutoImageProcessor.from_pretrained(name) | |
| net = AutoModelForDepthEstimation.from_pretrained(name).to(device).eval() | |
| o = net(**{k: v.to(device) for k, v in proc(images=Image.fromarray(img), return_tensors="pt").items()}).predicted_depth | |
| d = F.interpolate(o[None], size=img.shape[:2], mode="bicubic", align_corners=False)[0, 0].cpu().numpy() | |
| d = (d - d.min()) / max(float(d.max() - d.min()), 1e-6) | |
| return np.repeat((d * 255).round().astype(np.uint8)[..., None], 3, axis=2) | |
| def structure_map(img, task, device="cuda"): | |
| return {"hed": hed_map, "depth": depth_map}[task](img, device) | |