2D-3D / AnchorUDF /lib /data /TrainDatasetDF3D.py
Timilehin's picture
Upload folder using huggingface_hub
7eacb2e verified
Raw History Blame Contribute Delete
13.1 kB
from torch.utils.data import Dataset
import numpy as np
import os
import random
import torchvision.transforms as transforms
from PIL import Image, ImageOps
import torch
from PIL.ImageFilter import GaussianBlur
import trimesh
import logging
log = logging.getLogger('trimesh')
log.setLevel(40)
def load_trimesh(root_dir):
folders = os.listdir(root_dir)
meshs = {}
for i, f in enumerate(folders):
sub_name = f
meshs[sub_name] = trimesh.load(os.path.join(root_dir, f, '%s.obj' % (sub_name+'.obj_scaled')))
return meshs
class TrainDatasetDF3D(Dataset):
@staticmethod
def modify_commandline_options(parser, is_train):
return parser
def __init__(self, opt, phase='train'):
self.opt = opt
self.projection_mode = 'orthogonal'
# Path setup
self.root = self.opt.dataroot
self.RENDER = os.path.join(self.root, 'RENDER')
self.MASK = os.path.join(self.root, 'MASK')
self.PARAM = os.path.join(self.root, 'PARAM')
self.OBJ = os.path.join(self.root, 'GEO', 'OBJ')
self.TARGET = os.path.join(self.root, 'TARGET')
self.KEY_POINT = os.path.join(self.root, 'KEY_POINT')
self.B_MIN = np.array([-0.5, -0.5, -0.5])
self.B_MAX = np.array([0.5, 0.5, 0.5])
self.is_train = (phase == 'train')
self.load_size = self.opt.loadSize
self.num_views = self.opt.num_views
self.num_sample_inout = self.opt.num_sample_inout
self.num_key_points = self.opt.num_anchor_points
self.sample_distribution = np.array(self.opt.sample_distribution)
self.sample_sigmas = np.array(self.opt.sigma)
assert len(self.sample_distribution) == len(self.sample_sigmas)
self.num_samples = np.rint(self.sample_distribution * self.num_sample_inout).astype(np.uint32)
self.yaw_list = [0, 8, 16, 24, 32, 40, 48, 56, 64, 288, 296, 304, 312, 320, 328, 336, 344, 352]
self.pitch_list = [0]
self.subjects = self.get_subjects()
print(len(self.subjects))
# PIL to tensor
self.to_tensor = transforms.Compose([
transforms.Resize(self.load_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
# augmentation
self.aug_trans = transforms.Compose([
transforms.ColorJitter(brightness=opt.aug_bri, contrast=opt.aug_con, saturation=opt.aug_sat,
hue=opt.aug_hue)
])
def get_subjects(self):
all_subjects = os.listdir(self.RENDER)
var_subjects = np.loadtxt(os.path.join(self.root, 'val.txt'), dtype=str)
if len(var_subjects) == 0:
return all_subjects
if self.is_train:
return sorted(list(set(all_subjects) - set(var_subjects)))
else:
return sorted(list(var_subjects))
def __len__(self):
return len(self.subjects) * len(self.yaw_list) * len(self.pitch_list)
def get_render(self, subject, num_views, yid=0, pid=0, random_sample=False):
'''
Return the render data
:param subject: subject name
:param num_views: how many views to return
:param view_id: the first view_id. If None, select a random one.
:return:
'img': [num_views, C, W, H] images
'calib': [num_views, 4, 4] calibration matrix
'extrinsic': [num_views, 4, 4] extrinsic matrix
'mask': [num_views, 1, W, H] masks
'''
pitch = self.pitch_list[pid]
# The ids are an even distribution of num_views around view_id
view_ids = [self.yaw_list[(yid + len(self.yaw_list) // num_views * offset) % len(self.yaw_list)]
for offset in range(num_views)]
if random_sample:
view_ids = np.random.choice(self.yaw_list, num_views, replace=False)
calib_list = []
render_list = []
mask_list = []
extrinsic_list = []
scale_intrinsic_list = []
for vid in view_ids:
param_path = os.path.join(self.PARAM, subject, '%d_%d_%02d.npy' % (vid, pitch, 0))
render_path = os.path.join(self.RENDER, subject, '%d_%d_%02d.jpg' % (vid, pitch, 0))
mask_path = os.path.join(self.MASK, subject, '%d_%d_%02d.png' % (vid, pitch, 0))
# loading calibration data
param = np.load(param_path, allow_pickle=True)
# pixel unit / world unit
ortho_ratio = param.item().get('ortho_ratio')
# world unit / model unit
scale = param.item().get('scale')
# camera center world coordinate
center = param.item().get('center')
# model rotation
R = param.item().get('R')
translate = -np.matmul(R, center).reshape(3, 1)
extrinsic = np.concatenate([R, translate], axis=1)
extrinsic = np.concatenate([extrinsic, np.array([0, 0, 0, 1]).reshape(1, 4)], 0)
# Match camera space to image pixel space
scale_intrinsic = np.identity(4)
scale_intrinsic[0, 0] = scale / ortho_ratio
scale_intrinsic[1, 1] = -scale / ortho_ratio
scale_intrinsic[2, 2] = scale / ortho_ratio
# Match image pixel space to image uv space
uv_intrinsic = np.identity(4)
uv_intrinsic[0, 0] = 1.0 / float(self.opt.loadSize // 2)
uv_intrinsic[1, 1] = 1.0 / float(self.opt.loadSize // 2)
uv_intrinsic[2, 2] = 1.0 / float(self.opt.loadSize // 2)
# Transform under image pixel space
trans_intrinsic = np.identity(4)
mask = Image.open(mask_path).convert('L')
render = Image.open(render_path).convert('RGB')
if self.is_train:
# Pad images
pad_size = int(0.1 * self.load_size)
render = ImageOps.expand(render, pad_size, fill=0)
mask = ImageOps.expand(mask, pad_size, fill=0)
w, h = render.size
th, tw = self.load_size, self.load_size
# random flip
if self.opt.random_flip and np.random.rand() > 0.5:
scale_intrinsic[0, 0] *= -1
render = transforms.RandomHorizontalFlip(p=1.0)(render)
mask = transforms.RandomHorizontalFlip(p=1.0)(mask)
# random scale
if self.opt.random_scale:
rand_scale = random.uniform(0.9, 1.1)
w = int(rand_scale * w)
h = int(rand_scale * h)
render = render.resize((w, h), Image.BILINEAR)
mask = mask.resize((w, h), Image.NEAREST)
scale_intrinsic *= rand_scale
# random translate in the pixel space
if self.opt.random_trans:
dx = random.randint(-int(round((w - tw) / 10.)),
int(round((w - tw) / 10.)))
dy = random.randint(-int(round((h - th) / 10.)),
int(round((h - th) / 10.)))
else:
dx = 0
dy = 0
trans_intrinsic[0, 3] = -dx / float(self.opt.loadSize // 2)
trans_intrinsic[1, 3] = -dy / float(self.opt.loadSize // 2)
x1 = int(round((w - tw) / 2.)) + dx
y1 = int(round((h - th) / 2.)) + dy
render = render.crop((x1, y1, x1 + tw, y1 + th))
mask = mask.crop((x1, y1, x1 + tw, y1 + th))
render = self.aug_trans(render)
# random blur
if self.opt.aug_blur > 0.00001:
blur = GaussianBlur(np.random.uniform(0, self.opt.aug_blur))
render = render.filter(blur)
intrinsic = np.matmul(trans_intrinsic, np.matmul(uv_intrinsic, scale_intrinsic))
calib = torch.Tensor(np.matmul(intrinsic, extrinsic)).float()
extrinsic = torch.Tensor(extrinsic).float()
mask = transforms.Resize(self.load_size)(mask)
mask = transforms.ToTensor()(mask).float()
mask_list.append(mask)
render = self.to_tensor(render)
render = mask.expand_as(render) * render
render_list.append(render)
calib_list.append(calib)
extrinsic_list.append(extrinsic)
scale_intrinsic_list.append(scale_intrinsic)
return {
'img': torch.stack(render_list, dim=0),
'calib': torch.stack(calib_list, dim=0),
'extrinsic': torch.stack(extrinsic_list, dim=0),
'scale_intrinsic': abs(scale_intrinsic_list[0][0][0]) / float(self.opt.loadSize // 2),
'mask': torch.stack(mask_list, dim=0)
}
def select_sampling_method(self, subject, scale_intrinsic):
if not self.is_train:
random.seed(1991)
np.random.seed(1991)
torch.manual_seed(1991)
key_points_path = os.path.join(self.KEY_POINT, subject, 'key_point_{}.npz'.format(self.num_key_points))
key_points_npz = np.load(key_points_path)
key_points = key_points_npz['kp']
key_points = key_points.T
key_points = torch.Tensor(key_points).float()
sample_points = []
distances = []
sample_neighbors = []
for i, num in enumerate(self.num_samples):
boundary_samples_path = os.path.join(self.TARGET, subject, 'boundary_{}_direct_samples.npz'.format(self.sample_sigmas[i]))
boundary_samples_npz = np.load(boundary_samples_path)
boundary_sample_points = boundary_samples_npz['points']
boundary_sample_df = boundary_samples_npz['df']
boundary_sample_neighbors = boundary_samples_npz['neighbors']
total_indices = np.arange(boundary_sample_points.shape[0])
np.random.shuffle(total_indices)
sample_indices = total_indices[:num]
sample_points.extend(boundary_sample_points[sample_indices])
distances.extend(boundary_sample_df[sample_indices])
sample_neighbors.extend(boundary_sample_neighbors[sample_indices])
assert len(sample_points) == self.num_sample_inout
assert len(distances) == self.num_sample_inout
samples = np.array(sample_points).T
distances = np.array(distances)
samples = torch.Tensor(samples).float()
labels = torch.Tensor(distances).float() * scale_intrinsic
if self.opt.grad_constraint:
neighbors = np.array(sample_neighbors).T
neighbors = torch.Tensor(neighbors).float()
else:
neighbors = None
if neighbors is not None:
return {
'samples': samples,
'labels': labels,
'key_points': key_points,
'neighbors': neighbors
}
else:
return {
'samples': samples,
'labels': labels,
'key_points': key_points
}
def get_item(self, index):
# In case of a missing file or IO error, switch to a random sample instead
sid = index % len(self.subjects)
tmp = index // len(self.subjects)
yid = tmp % len(self.yaw_list)
pid = tmp // len(self.yaw_list)
# name of the subject 'rp_xxxx_xxx'
subject = self.subjects[sid]
res = {
'name': subject,
'mesh_path': os.path.join(self.OBJ, subject + '.obj'),
'sid': sid,
'yid': yid,
'pid': pid,
'b_min': self.B_MIN,
'b_max': self.B_MAX,
}
render_data = self.get_render(subject, num_views=self.num_views, yid=yid, pid=pid,
random_sample=self.opt.random_multiview)
res.update(render_data)
if self.opt.num_sample_inout:
sample_data = self.select_sampling_method(subject, res['scale_intrinsic'])
res.update(sample_data)
# img = np.uint8((np.transpose(render_data['img'][0].numpy(), (1, 2, 0)) * 0.5 + 0.5)[:, :, ::-1] * 255.0).copy()
# rot = render_data['calib'][0,:3, :3]
# trans = render_data['calib'][0,:3, 3:4]
# # pts = torch.addmm(trans, rot, sample_data['samples'][:, sample_data['labels'][0] > 0.5]) # [3, N]
# # pts = torch.addmm(trans, rot, sample_data['samples']) # [3, N]
# # pts = 0.5 * (pts.numpy().T + 1.0) * render_data['img'].size(2)
# kps = torch.addmm(trans, rot, sample_data['key_points'])
# kps = 0.5 * (kps.numpy().T + 1.0) * render_data['img'].size(2)
# for p in kps:
# img = cv2.circle(img, (p[0], p[1]), 2, (0,255,0), -1)
# cv2.imwrite('test_calib_kp.jpg', img)
# # cv2.imshow('test', img)
# # cv2.waitKey(1)
return res
def __getitem__(self, index):
return self.get_item(index)