2D-3D / main1.py
Timilehin's picture
Update main1.py
76ea4df verified
Raw History Blame Contribute Delete
16.4 kB
from anchor_utils import *
import os
import torch
import cv2
import numpy as np
from PIL import Image
from flask import Flask, request, send_file, make_response
from io import BytesIO
from flask import Flask, request, jsonify, render_template
from rembg import remove
import subprocess
import shutil
# Set up paths
# image_dir = os.path.join(os.getcwd(), 'pifuhd', 'sample_images'
# output_dir = os.path.join(os.getcwd(), "pifuhd", "results")
ckpt_path = os.path.join(os.getcwd(), 'pifuhd', 'checkpoints', 'pifuhd.pt')
import io
# os.chdir(os.path.join(os.getcwd(), 'lightweight-human-pose-estimation.pytorch'))
print('cwd', os.getcwd())
print(os.listdir(os.getcwd()))
import sys
sys.path.append(os.path.join(os.getcwd(), 'lightweight-human-pose-estimation.pytorch'))
sys.path.append(os.path.join(os.getcwd(), 'AnchorUDF'))
# Import necessary modules from lightweight human pose estimation
from models.with_mobilenet import PoseEstimationWithMobileNet
from modules.keypoints import extract_keypoints, group_keypoints
from modules.load_state import load_state
from modules.pose import Pose
import demo
# Initialize Flask app
app = Flask(__name__)
UPLOAD_FOLDER = os.getenv('UPLOAD_FOLDER', '/tmp/sample_images')
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
if not os.path.exists(app.config['UPLOAD_FOLDER']):
os.makedirs(app.config['UPLOAD_FOLDER'])
image_dir = os.path.join(app.config['UPLOAD_FOLDER'])
print('img_dir', image_dir)
# Load pose estimation model
net = PoseEstimationWithMobileNet()
checkpoint = torch.load(os.path.join(os.getcwd(), "lightweight-human-pose-estimation.pytorch", "checkpoint_iter_370000.pth"), map_location='cpu')
load_state(net, checkpoint)
net = net.eval()
# Preprocess function to crop image
def get_rect(net, image_path, height_size=512):
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
img = cv2.imread(image_path, cv2.IMREAD_COLOR)
orig_img = img.copy()
heatmaps, pafs, scale, pad = demo.infer_fast(net, img, height_size, stride, upsample_ratio, cpu=True)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints):
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale
rects = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
valid_keypoints = []
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0:
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
valid_keypoints.append([pose_keypoints[kpt_id, 0], pose_keypoints[kpt_id, 1]])
valid_keypoints = np.array(valid_keypoints)
if pose_entries[n][10] != -1.0 or pose_entries[n][13] != -1.0:
pmin = valid_keypoints.min(0)
pmax = valid_keypoints.max(0)
center = (0.5 * (pmax[:2] + pmin[:2])).astype(np.int32)
radius = int(0.65 * max(pmax[0] - pmin[0], pmax[1] - pmin[1]))
elif pose_entries[n][10] == -1.0 and pose_entries[n][13] == -1.0 and pose_entries[n][8] != -1.0 and pose_entries[n][11] != -1.0:
center = (0.5 * (pose_keypoints[8] + pose_keypoints[11])).astype(np.int32)
radius = int(1.45 * np.sqrt(((center[None, :] - valid_keypoints) ** 2).sum(1)).max(0))
center[1] += int(0.05 * radius)
else:
center = np.array([img.shape[1] // 2, img.shape[0] // 2])
radius = max(img.shape[1] // 2, img.shape[0] // 2)
x1 = center[0] - radius
y1 = center[1] - radius
rects.append([x1, y1, 2 * radius, 2 * radius])
return np.array(rects)
from pifuhd.apps.simple_test import run_pifuhd
@torch.inference_mode()
def process_image_pifuhd(image):
# Define paths
# image_path = os.path.join(os.getcwd(), image_dir, "uploaded_image.png")
image_path = os.path.join(app.config['UPLOAD_FOLDER'], "uploaded_image.png")
obj_path = os.path.join(app.config['UPLOAD_FOLDER'], "output.obj")
# Save the uploaded image
image.save(image_path)
# Preprocess the image to get rect
rects = get_rect(net, image_path)
rect_path = image_path.replace('.png', '_rect.txt')
np.savetxt(rect_path, rects, fmt='%d')
# Run PIFuHD and get the buffer
obj_buffer = run_pifuhd(
input_path=image_dir,
ckpt_path=ckpt_path,
resolution=256,
use_rect=True
)
# # Save the first buffer as an OBJ file
with open(obj_path, 'wb') as f:
f.write(obj_buffer[0].getvalue())
return obj_buffer[0], obj_path
# @app.route('/human')
# def pifuhd_page():
# return render_template('pifuhd.html')
@app.route('/generate-mesh', methods=['POST'])
def generate_mesh():
if 'file' not in request.files:
return "No file part", 400
file = request.files['file']
if file.filename == '':
return "No selected file", 400
if file:
# Convert the file to an image and process it
image = Image.open(file.stream)
obj_buffer, _ = process_image_pifuhd(image)
# Create a BytesIO object to send the file
obj_bytes_io = io.BytesIO(obj_buffer.getvalue())
obj_bytes_io.seek(0) # Ensure the buffer is at the start
# Return the OBJ file in memory
return send_file(
obj_bytes_io,
as_attachment=True,
download_name='output.obj',
mimetype='application/octet-stream'
)
# Helper function for garment inference
@torch.inference_mode()
def run_garment_inference():
upload_folder = app.config['UPLOAD_FOLDER']
command = [
"python", "-m", "apps.eval", "--results_path", os.path.join(upload_folder, "AnchorUDF", "out"), "--name", "out",
"--dataroot", os.path.join(upload_folder, "AnchorUDF/pics/my_test"), "--test_folder_path", "1",
# "--load_netG_checkpoint_path", os.path.join(upload_folder, 'netG_epoch_59'),
"--load_netG_checkpoint_path", os.path.join(os.getcwd(), 'netG_epoch_59'),
"--anchor", "--num_steps", "2", "--filter_val", "0.0075",
"--max_dist", "0.05", "--num_gen_points", "850000", "--gamma", "1",
"--aug_bri", "1.5", "--aug_con", "0.5", "--aug_sat", "0.5", "--aug_hue", "0.5"
]
# subprocess.run(command, cwd=os.path.join(upload_folder, "AnchorUDF"))
subprocess.run(command, cwd=os.path.join(os.getcwd(), "AnchorUDF"))
print('garment generated!!!')
# Helper function to process images
def process_image(filename):
# input_path = f'{os.getcwd()}/sample_images/{filename}'
# input_path = filename
upload_folder = app.config['UPLOAD_FOLDER']
# output_path = f'{os.getcwd()}/AnchorUDF/pics/my_test/RENDER/1/0_0_00.jpg'
output_path = os.path.join(upload_folder, 'AnchorUDF/pics/my_test/RENDER/1/0_0_00.jpg')
input_image = Image.open(filename)
# Determine new size
if input_image.size[0] <= 600 or input_image.size[1] <= 600:
new_size = (500, 500)
elif input_image.size[0] >= 900 or input_image.size[1] >= 900:
new_size = (1500, 1500)
elif input_image.size[0] <= 900 or input_image.size[1] <= 900 and input_image.size[0] >= 700 or input_image.size[1] >= 700:
new_size = (1000, 1000)
else:
new_size = input_image.size
resized_image = input_image.resize(new_size, Image.LANCZOS)
output_image = remove(resized_image)
rgb_output_image = output_image.convert('RGB')
# Save the processed image
os.makedirs(os.path.dirname(output_path), exist_ok=True)
rgb_output_image.save(output_path, quality=200, optimize=True)
return output_path
# Helper function to generate mask
def generate_mask(image_path):
upload_folder = app.config['UPLOAD_FOLDER']
mask_dir = os.path.join(upload_folder, "AnchorUDF/pics/my_test/MASK/1")
# Create the directory if it doesn't exist
os.makedirs(mask_dir, exist_ok=True)
mask_path = os.path.join(mask_dir, "0_0_00.png")
image = cv2.imread(image_path)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
_, binary = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY)
contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
mask = np.zeros_like(gray)
cv2.drawContours(mask, contours, -1, 255, thickness=cv2.FILLED)
cv2.imwrite(mask_path, mask)
print('mask path', mask_path)
return mask_path
# Helper function for mesh processing and returning cleaned .ply file
def process_mesh(scale_factor=1.0, nb_neighbors=40, std_ratio=1.0, voxel_size=0.005):
off_path = None
# al_obj = glob.glob(f'{os.getcwd()}/AnchorUDF/out/out/*.off')
# # print('all_obg', al_obj)
upload_folder = app.config['UPLOAD_FOLDER']
print(upload_folder)
al_obj = glob.glob(os.path.join(upload_folder, "AnchorUDF", "out", "out", "*.off"))
print('all_obj', al_obj)
for i in al_obj:
if os.path.getsize(i) > 30000:
print('found off file', i)
off_path = i
break
if not off_path:
raise FileNotFoundError("No suitable OFF file found.")
filename = 'output'
saved_ply_path = f"{off_path}_{filename}.ply"
convert_off_to_ply(off_path, saved_ply_path, scale_factor)
cleaned_ply_path, cleaned_pcd_path = remove_outliers_and_convert_to_point_cloud(saved_ply_path, nb_neighbors, std_ratio, voxel_size)
obj_path = f"{cleaned_ply_path}_final.obj"
print('saving obj to ', obj_path)
obj_path_fpath = convert_pointcloud_to_mesh(cleaned_ply_path, obj_path)
print('cleaned ply path', cleaned_ply_path)
return cleaned_ply_path, obj_path_fpath
# import os
# import sys
# from apps.eval import Evaluator
# def run_garment_inference():
# print('garm working dir', os.getcwd())
# # Set up the options manually instead of passing them via command line arguments
# class Options:
# results_path = "out"
# name = "out"
# dataroot = os.path.join(os.getcwd(), 'AnchorUDF/pics/my_test')
# test_folder_path = "1"
# load_netG_checkpoint_path = os.path.join(os.getcwd(), 'netG_epoch_59')
# anchor = True
# num_steps = 2
# filter_val = 0.0075
# max_dist = 0.05
# num_gen_points = 850000
# gamma = 1
# aug_bri = 1.5
# aug_con = 0.5
# aug_sat = 0.5
# aug_hue = 0.5
# gpu_id = 0
# mgpu = False
# loadSize = 512 # or whatever size you need
# # Initialize the evaluator with the options
# opt = Options()
# evaluator = Evaluator(opt)
# # Run the evaluation
# yaw_list = [0]
# for vid in yaw_list:
# image_path = os.path.join(opt.dataroot, 'RENDER', opt.test_folder_path, f'{vid}_0_00.jpg')
# mask_path = os.path.join(opt.dataroot, 'MASK', opt.test_folder_path, f'{vid}_0_00.png')
# print(image_path, mask_path)
# data = evaluator.load_image(image_path, mask_path)
# evaluator.eval(data)
# print('garment generated!!!')
@app.route('/process', methods=['POST'])
def process():
if 'file' not in request.files:
return jsonify({"error": "No file part in the request"}), 400
file = request.files['file']
if file.filename == '':
return jsonify({"error": "No selected file"}), 400
# You can save the file temporarily if needed or process it directly
# temp_filename = os.path.join('/tmp', file.filename)
# file.save(temp_filename)
# Process the image directly from the file stream
processed_image_path = process_image(file.stream)
mask_path = generate_mask(processed_image_path)
run_garment_inference()
try:
print('generating ply...')
cleaned_ply_path, _ = process_mesh(scale_factor=0.5, nb_neighbors=40, std_ratio=1.0, voxel_size=0.003)
print('ply generated')
except FileNotFoundError as e:
import traceback
traceback.print_exc()
return jsonify({"error": str(e)}), 500
return send_file(cleaned_ply_path, as_attachment=True)
@app.route('/anchor')
def anchorudf_page():
return render_template('anchor.html')
# Set your paths in the configuration
app.config['HUMAN_MODEL_PATH'] = os.path.join(app.config['UPLOAD_FOLDER'])
app.config['GARMENT_MODEL_PATH'] = os.path.join(app.config['UPLOAD_FOLDER'], "AnchorUDF", "out", "out")
@app.route('/process_3d_models', methods=['GET'])
def process_3d_models():
# Get the file paths
try:
import bpy
human_m = glob.glob(os.path.join(app.config['HUMAN_MODEL_PATH'], "output.obj"))[0]
garment_m = glob.glob(os.path.join(app.config['GARMENT_MODEL_PATH'], '*.obj'))[-1]
print(f'draping {garment_m} on {human_m}')
# Get the file sizes in bytes
human_m_size_bytes = os.path.getsize(human_m)
garment_m_size_bytes = os.path.getsize(garment_m)
# Convert bytes to MB
human_m_size_mb = human_m_size_bytes / (1024 * 1024)
garment_m_size_mb = garment_m_size_bytes / (1024 * 1024)
# Clear existing Blender data
bpy.ops.wm.read_factory_settings(use_empty=True)
# Import the human model
bpy.ops.import_scene.obj(filepath=human_m)
human = bpy.context.selected_objects[0]
# Import the garment model
bpy.ops.import_scene.obj(filepath=garment_m)
garment = bpy.context.selected_objects[0]
# Align and scale the garment model to the human model
human_dimensions = human.dimensions
garment_dimensions = garment.dimensions
# Scale factor for uniform scaling
uniform_scale_factor = min(
human_dimensions.x / garment_dimensions.x,
human_dimensions.y / garment_dimensions.y,
human_dimensions.z / garment_dimensions.z
) + 0.1
garment.scale = (uniform_scale_factor, uniform_scale_factor, uniform_scale_factor)
garment.location = human.location
garment.location.z += 0.35
# Apply transformations
bpy.context.view_layer.objects.active = garment
bpy.ops.object.transform_apply(location=True, scale=True, rotation=True)
# Add cloth modifier to garment
garment.select_set(True)
bpy.ops.object.modifier_add(type='CLOTH')
# Add collision to human model
human.select_set(True)
bpy.ops.object.modifier_add(type='COLLISION')
# Bake the cloth simulation
bpy.context.view_layer.objects.active = garment
# bpy.ops.ptcache.bake_all(bake=True)
# Export the fitted garment model
output_obj_path = os.path.join(app.config['UPLOAD_FOLDER'], 'fitted_model.obj')
bpy.ops.export_scene.obj(filepath=output_obj_path, use_selection=True)
# Response with file paths and sizes
# return jsonify({
# "human_model_path": human_m,
# "human_model_size_mb": human_m_size_mb,
# "garment_model_path": garment_m,
# "garment_model_size_mb": garment_m_size_mb,
# "output_obj_path": output_obj_path
# })
return send_file(output_obj_path, as_attachment=True)
except Exception as e:
raise e
return jsonify({'error': str(e)}), 500
# @app.route('/')
# def index():
# return render_template('index.html')
# Route for Human Mesh Generator page
@app.route('/')
def human():
return render_template('human.html')
# Route for Garment Processing page
@app.route('/garment')
def garment():
return render_template('garment.html')
# Route for Draping Garment on Human Model page
@app.route('/drape')
def drape():
return render_template('drape.html')
# Run the Flask app
if __name__ == "__main__":
app.run(host='0.0.0.0', port=6100)