File size: 2,670 Bytes
620a865
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import torch
import torch.nn.functional as F
import torchvision.transforms as T
from bert.modeling_bert import BertModel
from lib import segmentation
import numpy as np


class FaceSegment:
    def __init__(
        self,
        model_path: str,
        embedding_path: str = None,
        device: str = "cpu",
        img_size: int = 480,
    ):
        self.device = torch.device(device)

        self.transform = T.Compose([
            T.Resize(img_size),
            T.ToTensor(),
            T.Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225]
            )
        ])

        token_ids = [101, 2529, 2132, 102]
        max_len   = 20
        padded    = token_ids + [0] * (max_len - len(token_ids))
        attn_mask = [1] * len(token_ids) + [0] * (max_len - len(token_ids))

        self.sent_tensor  = torch.tensor([padded], dtype=torch.long,  device=self.device)
        self.mask_tensor  = torch.tensor([attn_mask], dtype=torch.long, device=self.device)
        self.l_mask4seg   = self.mask_tensor.unsqueeze(-1)

        class Opts:
            swin_type = "base"
            window12  = True
            mha        = ""
            fusion_drop= 0.0

        self.model_seg = segmentation.lavt(pretrained="", args=Opts())
        self.model_seg.load_state_dict(torch.load(model_path, map_location="cpu")["model"])
        self.model_seg.to(self.device).eval()

        if embedding_path:
            emb_data = torch.load(embedding_path, map_location=self.device)
            self.embedding = emb_data["embedding"].to(self.device)
        else:
            bert_ckpt = torch.load(model_path, map_location="cpu")["bert_model"]
            self.bert = BertModel.from_pretrained("bert-base-uncased")
            self.bert.pooler = None
            self.bert.load_state_dict(bert_ckpt)
            self.bert.to(self.device).eval()
            with torch.no_grad():
                last_hidden = self.bert(self.sent_tensor, attention_mask=self.mask_tensor)[0]
                self.embedding = last_hidden.permute(0, 2, 1).contiguous()

    def get_face(self, image_pil):
        orig_w, orig_h = image_pil.size
        x = self.transform(image_pil).unsqueeze(0).to(self.device)

        with torch.no_grad():
            out = self.model_seg(x, self.embedding, l_mask=self.l_mask4seg)
            out = out.argmax(dim=1, keepdim=True).float()
            out = F.interpolate(out, size=(orig_h, orig_w), mode="nearest")

        mask = (out.squeeze(0).squeeze(0).cpu().numpy() > 0).astype(np.uint8) * 255
        rgb  = np.array(image_pil, dtype=np.uint8)
        rgba = np.dstack([rgb, mask])

        return rgba