Spaces:
Sleeping
Sleeping
Upload util/pos_embed.py with huggingface_hub
Browse files- util/pos_embed.py +92 -0
util/pos_embed.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
# Partly revised by YZ @UCL&Moorfields
|
| 4 |
+
# --------------------------------------------------------
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
# 2D sine-cosine position embedding
|
| 12 |
+
# References:
|
| 13 |
+
# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py
|
| 14 |
+
# MoCo v3: https://github.com/facebookresearch/moco-v3
|
| 15 |
+
# --------------------------------------------------------
|
| 16 |
+
def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
|
| 17 |
+
"""
|
| 18 |
+
grid_size: int of the grid height and width
|
| 19 |
+
return:
|
| 20 |
+
pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
|
| 21 |
+
"""
|
| 22 |
+
grid_h = np.arange(grid_size, dtype=np.float32)
|
| 23 |
+
grid_w = np.arange(grid_size, dtype=np.float32)
|
| 24 |
+
grid = np.meshgrid(grid_w, grid_h) # here w goes first
|
| 25 |
+
grid = np.stack(grid, axis=0)
|
| 26 |
+
|
| 27 |
+
grid = grid.reshape([2, 1, grid_size, grid_size])
|
| 28 |
+
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
|
| 29 |
+
if cls_token:
|
| 30 |
+
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
|
| 31 |
+
return pos_embed
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
|
| 35 |
+
assert embed_dim % 2 == 0
|
| 36 |
+
|
| 37 |
+
# use half of dimensions to encode grid_h
|
| 38 |
+
emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
|
| 39 |
+
emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
|
| 40 |
+
|
| 41 |
+
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
| 42 |
+
return emb
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
| 46 |
+
"""
|
| 47 |
+
embed_dim: output dimension for each position
|
| 48 |
+
pos: a list of positions to be encoded: size (M,)
|
| 49 |
+
out: (M, D)
|
| 50 |
+
"""
|
| 51 |
+
assert embed_dim % 2 == 0
|
| 52 |
+
omega = np.arange(embed_dim // 2, dtype=np.float)
|
| 53 |
+
omega /= embed_dim / 2.
|
| 54 |
+
omega = 1. / 10000**omega # (D/2,)
|
| 55 |
+
|
| 56 |
+
pos = pos.reshape(-1) # (M,)
|
| 57 |
+
out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
|
| 58 |
+
|
| 59 |
+
emb_sin = np.sin(out) # (M, D/2)
|
| 60 |
+
emb_cos = np.cos(out) # (M, D/2)
|
| 61 |
+
|
| 62 |
+
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
| 63 |
+
return emb
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# --------------------------------------------------------
|
| 67 |
+
# Interpolate position embeddings for high-resolution
|
| 68 |
+
# References:
|
| 69 |
+
# DeiT: https://github.com/facebookresearch/deit
|
| 70 |
+
# --------------------------------------------------------
|
| 71 |
+
def interpolate_pos_embed(model, checkpoint_model):
|
| 72 |
+
if 'pos_embed' in checkpoint_model:
|
| 73 |
+
pos_embed_checkpoint = checkpoint_model['pos_embed']
|
| 74 |
+
embedding_size = pos_embed_checkpoint.shape[-1]
|
| 75 |
+
num_patches = model.patch_embed.num_patches
|
| 76 |
+
num_extra_tokens = model.pos_embed.shape[-2] - num_patches
|
| 77 |
+
# height (== width) for the checkpoint position embedding
|
| 78 |
+
orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
|
| 79 |
+
# height (== width) for the new position embedding
|
| 80 |
+
new_size = int(num_patches ** 0.5)
|
| 81 |
+
# class_token and dist_token are kept unchanged
|
| 82 |
+
if orig_size != new_size:
|
| 83 |
+
print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size))
|
| 84 |
+
extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
|
| 85 |
+
# only the position tokens are interpolated
|
| 86 |
+
pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
|
| 87 |
+
pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
|
| 88 |
+
pos_tokens = torch.nn.functional.interpolate(
|
| 89 |
+
pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
|
| 90 |
+
pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
|
| 91 |
+
new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
|
| 92 |
+
checkpoint_model['pos_embed'] = new_pos_embed
|