File size: 1,766 Bytes
6b9d0d8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | # mtmd-debug
## Debugging encode pass
Example of debugging an input gray image (raw, not preprocessed):
```py
from transformers import AutoModel
model = AutoModel.from_pretrained(...)
def test_vision():
img_size = 896 # number of patches per side
pixel_values = torch.zeros(1, 3, img_size, img_size) + 0.5 # gray image
with torch.no_grad():
outputs = model.model.get_image_features(pixel_values=pixel_values)
print("last_hidden_state shape:", outputs.last_hidden_state.shape)
print("last_hidden_state:", outputs.last_hidden_state)
test_vision()
```
Example of debugging a rainbow image:
```py
import torch
import math
def make_rainbow(img_size):
cx, cy = img_size / 2.0, img_size / 2.0
max_dist = math.sqrt(cx * cx + cy * cy)
img = torch.zeros(1, 3, img_size, img_size)
for y in range(img_size):
for x in range(img_size):
dx, dy = x - cx, y - cy
hue = math.atan2(dy, dx) / (2 * math.pi)
if hue < 0:
hue += 1
sat = math.sqrt(dx * dx + dy * dy) / max_dist
sat = min(sat, 1.0)
h6 = hue * 6
i6 = int(h6)
f = h6 - i6
p = 1 - sat
q = 1 - sat * f
t = 1 - sat * (1 - f)
rgb = [(1,t,p),(q,1,p),(p,1,t),(p,q,1),(t,p,1),(1,p,q)][i6 % 6]
img[0, 0, y, x] = rgb[0]
img[0, 1, y, x] = rgb[1]
img[0, 2, y, x] = rgb[2]
return img
img_size = 896
pixel_values = make_rainbow(img_size)
with torch.no_grad():
outputs = model.model.get_image_features(pixel_values=pixel_values)
print("last_hidden_state:", outputs.last_hidden_state)
```
## Debugging preprocess pass
(TODO)
|