janvib commited on
Commit
b3ebe8c
·
verified ·
1 Parent(s): 961ee08

Upload util/pos_embed.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. util/pos_embed.py +92 -0
util/pos_embed.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ # Partly revised by YZ @UCL&Moorfields
4
+ # --------------------------------------------------------
5
+
6
+ import numpy as np
7
+
8
+ import torch
9
+
10
+ # --------------------------------------------------------
11
+ # 2D sine-cosine position embedding
12
+ # References:
13
+ # Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py
14
+ # MoCo v3: https://github.com/facebookresearch/moco-v3
15
+ # --------------------------------------------------------
16
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
17
+ """
18
+ grid_size: int of the grid height and width
19
+ return:
20
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
21
+ """
22
+ grid_h = np.arange(grid_size, dtype=np.float32)
23
+ grid_w = np.arange(grid_size, dtype=np.float32)
24
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
25
+ grid = np.stack(grid, axis=0)
26
+
27
+ grid = grid.reshape([2, 1, grid_size, grid_size])
28
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
29
+ if cls_token:
30
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
31
+ return pos_embed
32
+
33
+
34
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
35
+ assert embed_dim % 2 == 0
36
+
37
+ # use half of dimensions to encode grid_h
38
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
39
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
40
+
41
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
42
+ return emb
43
+
44
+
45
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
46
+ """
47
+ embed_dim: output dimension for each position
48
+ pos: a list of positions to be encoded: size (M,)
49
+ out: (M, D)
50
+ """
51
+ assert embed_dim % 2 == 0
52
+ omega = np.arange(embed_dim // 2, dtype=np.float)
53
+ omega /= embed_dim / 2.
54
+ omega = 1. / 10000**omega # (D/2,)
55
+
56
+ pos = pos.reshape(-1) # (M,)
57
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
58
+
59
+ emb_sin = np.sin(out) # (M, D/2)
60
+ emb_cos = np.cos(out) # (M, D/2)
61
+
62
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
63
+ return emb
64
+
65
+
66
+ # --------------------------------------------------------
67
+ # Interpolate position embeddings for high-resolution
68
+ # References:
69
+ # DeiT: https://github.com/facebookresearch/deit
70
+ # --------------------------------------------------------
71
+ def interpolate_pos_embed(model, checkpoint_model):
72
+ if 'pos_embed' in checkpoint_model:
73
+ pos_embed_checkpoint = checkpoint_model['pos_embed']
74
+ embedding_size = pos_embed_checkpoint.shape[-1]
75
+ num_patches = model.patch_embed.num_patches
76
+ num_extra_tokens = model.pos_embed.shape[-2] - num_patches
77
+ # height (== width) for the checkpoint position embedding
78
+ orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
79
+ # height (== width) for the new position embedding
80
+ new_size = int(num_patches ** 0.5)
81
+ # class_token and dist_token are kept unchanged
82
+ if orig_size != new_size:
83
+ print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size))
84
+ extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
85
+ # only the position tokens are interpolated
86
+ pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
87
+ pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
88
+ pos_tokens = torch.nn.functional.interpolate(
89
+ pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
90
+ pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
91
+ new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
92
+ checkpoint_model['pos_embed'] = new_pos_embed