from anchor_utils import * import os import torch import cv2 import numpy as np from PIL import Image from flask import Flask, request, send_file, make_response from io import BytesIO from flask import Flask, request, jsonify, render_template from rembg import remove import subprocess import shutil # Set up paths # image_dir = os.path.join(os.getcwd(), 'pifuhd', 'sample_images' # output_dir = os.path.join(os.getcwd(), "pifuhd", "results") ckpt_path = os.path.join(os.getcwd(), 'pifuhd', 'checkpoints', 'pifuhd.pt') import io # os.chdir(os.path.join(os.getcwd(), 'lightweight-human-pose-estimation.pytorch')) print('cwd', os.getcwd()) print(os.listdir(os.getcwd())) import sys sys.path.append(os.path.join(os.getcwd(), 'lightweight-human-pose-estimation.pytorch')) sys.path.append(os.path.join(os.getcwd(), 'AnchorUDF')) # Import necessary modules from lightweight human pose estimation from models.with_mobilenet import PoseEstimationWithMobileNet from modules.keypoints import extract_keypoints, group_keypoints from modules.load_state import load_state from modules.pose import Pose import demo # Initialize Flask app app = Flask(__name__) UPLOAD_FOLDER = os.getenv('UPLOAD_FOLDER', '/tmp/sample_images') app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER if not os.path.exists(app.config['UPLOAD_FOLDER']): os.makedirs(app.config['UPLOAD_FOLDER']) image_dir = os.path.join(app.config['UPLOAD_FOLDER']) print('img_dir', image_dir) # Load pose estimation model net = PoseEstimationWithMobileNet() checkpoint = torch.load(os.path.join(os.getcwd(), "lightweight-human-pose-estimation.pytorch", "checkpoint_iter_370000.pth"), map_location='cpu') load_state(net, checkpoint) net = net.eval() # Preprocess function to crop image def get_rect(net, image_path, height_size=512): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts img = cv2.imread(image_path, cv2.IMREAD_COLOR) orig_img = img.copy() heatmaps, pafs, scale, pad = demo.infer_fast(net, img, height_size, stride, upsample_ratio, cpu=True) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale rects = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 valid_keypoints = [] for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) valid_keypoints.append([pose_keypoints[kpt_id, 0], pose_keypoints[kpt_id, 1]]) valid_keypoints = np.array(valid_keypoints) if pose_entries[n][10] != -1.0 or pose_entries[n][13] != -1.0: pmin = valid_keypoints.min(0) pmax = valid_keypoints.max(0) center = (0.5 * (pmax[:2] + pmin[:2])).astype(np.int32) radius = int(0.65 * max(pmax[0] - pmin[0], pmax[1] - pmin[1])) elif pose_entries[n][10] == -1.0 and pose_entries[n][13] == -1.0 and pose_entries[n][8] != -1.0 and pose_entries[n][11] != -1.0: center = (0.5 * (pose_keypoints[8] + pose_keypoints[11])).astype(np.int32) radius = int(1.45 * np.sqrt(((center[None, :] - valid_keypoints) ** 2).sum(1)).max(0)) center[1] += int(0.05 * radius) else: center = np.array([img.shape[1] // 2, img.shape[0] // 2]) radius = max(img.shape[1] // 2, img.shape[0] // 2) x1 = center[0] - radius y1 = center[1] - radius rects.append([x1, y1, 2 * radius, 2 * radius]) return np.array(rects) from pifuhd.apps.simple_test import run_pifuhd @torch.inference_mode() def process_image_pifuhd(image): # Define paths # image_path = os.path.join(os.getcwd(), image_dir, "uploaded_image.png") image_path = os.path.join(app.config['UPLOAD_FOLDER'], "uploaded_image.png") obj_path = os.path.join(app.config['UPLOAD_FOLDER'], "output.obj") # Save the uploaded image image.save(image_path) # Preprocess the image to get rect rects = get_rect(net, image_path) rect_path = image_path.replace('.png', '_rect.txt') np.savetxt(rect_path, rects, fmt='%d') # Run PIFuHD and get the buffer obj_buffer = run_pifuhd( input_path=image_dir, ckpt_path=ckpt_path, resolution=256, use_rect=True ) # # Save the first buffer as an OBJ file with open(obj_path, 'wb') as f: f.write(obj_buffer[0].getvalue()) return obj_buffer[0], obj_path # @app.route('/human') # def pifuhd_page(): # return render_template('pifuhd.html') @app.route('/generate-mesh', methods=['POST']) def generate_mesh(): if 'file' not in request.files: return "No file part", 400 file = request.files['file'] if file.filename == '': return "No selected file", 400 if file: # Convert the file to an image and process it image = Image.open(file.stream) obj_buffer, _ = process_image_pifuhd(image) # Create a BytesIO object to send the file obj_bytes_io = io.BytesIO(obj_buffer.getvalue()) obj_bytes_io.seek(0) # Ensure the buffer is at the start # Return the OBJ file in memory return send_file( obj_bytes_io, as_attachment=True, download_name='output.obj', mimetype='application/octet-stream' ) # Helper function for garment inference @torch.inference_mode() def run_garment_inference(): upload_folder = app.config['UPLOAD_FOLDER'] command = [ "python", "-m", "apps.eval", "--results_path", os.path.join(upload_folder, "AnchorUDF", "out"), "--name", "out", "--dataroot", os.path.join(upload_folder, "AnchorUDF/pics/my_test"), "--test_folder_path", "1", # "--load_netG_checkpoint_path", os.path.join(upload_folder, 'netG_epoch_59'), "--load_netG_checkpoint_path", os.path.join(os.getcwd(), 'netG_epoch_59'), "--anchor", "--num_steps", "2", "--filter_val", "0.0075", "--max_dist", "0.05", "--num_gen_points", "850000", "--gamma", "1", "--aug_bri", "1.5", "--aug_con", "0.5", "--aug_sat", "0.5", "--aug_hue", "0.5" ] # subprocess.run(command, cwd=os.path.join(upload_folder, "AnchorUDF")) subprocess.run(command, cwd=os.path.join(os.getcwd(), "AnchorUDF")) print('garment generated!!!') # Helper function to process images def process_image(filename): # input_path = f'{os.getcwd()}/sample_images/{filename}' # input_path = filename upload_folder = app.config['UPLOAD_FOLDER'] # output_path = f'{os.getcwd()}/AnchorUDF/pics/my_test/RENDER/1/0_0_00.jpg' output_path = os.path.join(upload_folder, 'AnchorUDF/pics/my_test/RENDER/1/0_0_00.jpg') input_image = Image.open(filename) # Determine new size if input_image.size[0] <= 600 or input_image.size[1] <= 600: new_size = (500, 500) elif input_image.size[0] >= 900 or input_image.size[1] >= 900: new_size = (1500, 1500) elif input_image.size[0] <= 900 or input_image.size[1] <= 900 and input_image.size[0] >= 700 or input_image.size[1] >= 700: new_size = (1000, 1000) else: new_size = input_image.size resized_image = input_image.resize(new_size, Image.LANCZOS) output_image = remove(resized_image) rgb_output_image = output_image.convert('RGB') # Save the processed image os.makedirs(os.path.dirname(output_path), exist_ok=True) rgb_output_image.save(output_path, quality=200, optimize=True) return output_path # Helper function to generate mask def generate_mask(image_path): upload_folder = app.config['UPLOAD_FOLDER'] mask_dir = os.path.join(upload_folder, "AnchorUDF/pics/my_test/MASK/1") # Create the directory if it doesn't exist os.makedirs(mask_dir, exist_ok=True) mask_path = os.path.join(mask_dir, "0_0_00.png") image = cv2.imread(image_path) gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) _, binary = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY) contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) mask = np.zeros_like(gray) cv2.drawContours(mask, contours, -1, 255, thickness=cv2.FILLED) cv2.imwrite(mask_path, mask) print('mask path', mask_path) return mask_path # Helper function for mesh processing and returning cleaned .ply file def process_mesh(scale_factor=1.0, nb_neighbors=40, std_ratio=1.0, voxel_size=0.005): off_path = None # al_obj = glob.glob(f'{os.getcwd()}/AnchorUDF/out/out/*.off') # # print('all_obg', al_obj) upload_folder = app.config['UPLOAD_FOLDER'] print(upload_folder) al_obj = glob.glob(os.path.join(upload_folder, "AnchorUDF", "out", "out", "*.off")) print('all_obj', al_obj) for i in al_obj: if os.path.getsize(i) > 30000: print('found off file', i) off_path = i break if not off_path: raise FileNotFoundError("No suitable OFF file found.") filename = 'output' saved_ply_path = f"{off_path}_{filename}.ply" convert_off_to_ply(off_path, saved_ply_path, scale_factor) cleaned_ply_path, cleaned_pcd_path = remove_outliers_and_convert_to_point_cloud(saved_ply_path, nb_neighbors, std_ratio, voxel_size) obj_path = f"{cleaned_ply_path}_final.obj" print('saving obj to ', obj_path) obj_path_fpath = convert_pointcloud_to_mesh(cleaned_ply_path, obj_path) print('cleaned ply path', cleaned_ply_path) return cleaned_ply_path, obj_path_fpath # import os # import sys # from apps.eval import Evaluator # def run_garment_inference(): # print('garm working dir', os.getcwd()) # # Set up the options manually instead of passing them via command line arguments # class Options: # results_path = "out" # name = "out" # dataroot = os.path.join(os.getcwd(), 'AnchorUDF/pics/my_test') # test_folder_path = "1" # load_netG_checkpoint_path = os.path.join(os.getcwd(), 'netG_epoch_59') # anchor = True # num_steps = 2 # filter_val = 0.0075 # max_dist = 0.05 # num_gen_points = 850000 # gamma = 1 # aug_bri = 1.5 # aug_con = 0.5 # aug_sat = 0.5 # aug_hue = 0.5 # gpu_id = 0 # mgpu = False # loadSize = 512 # or whatever size you need # # Initialize the evaluator with the options # opt = Options() # evaluator = Evaluator(opt) # # Run the evaluation # yaw_list = [0] # for vid in yaw_list: # image_path = os.path.join(opt.dataroot, 'RENDER', opt.test_folder_path, f'{vid}_0_00.jpg') # mask_path = os.path.join(opt.dataroot, 'MASK', opt.test_folder_path, f'{vid}_0_00.png') # print(image_path, mask_path) # data = evaluator.load_image(image_path, mask_path) # evaluator.eval(data) # print('garment generated!!!') @app.route('/process', methods=['POST']) def process(): if 'file' not in request.files: return jsonify({"error": "No file part in the request"}), 400 file = request.files['file'] if file.filename == '': return jsonify({"error": "No selected file"}), 400 # You can save the file temporarily if needed or process it directly # temp_filename = os.path.join('/tmp', file.filename) # file.save(temp_filename) # Process the image directly from the file stream processed_image_path = process_image(file.stream) mask_path = generate_mask(processed_image_path) run_garment_inference() try: print('generating ply...') cleaned_ply_path, _ = process_mesh(scale_factor=0.5, nb_neighbors=40, std_ratio=1.0, voxel_size=0.003) print('ply generated') except FileNotFoundError as e: import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 return send_file(cleaned_ply_path, as_attachment=True) @app.route('/anchor') def anchorudf_page(): return render_template('anchor.html') # Set your paths in the configuration app.config['HUMAN_MODEL_PATH'] = os.path.join(app.config['UPLOAD_FOLDER']) app.config['GARMENT_MODEL_PATH'] = os.path.join(app.config['UPLOAD_FOLDER'], "AnchorUDF", "out", "out") @app.route('/process_3d_models', methods=['GET']) def process_3d_models(): # Get the file paths try: import bpy human_m = glob.glob(os.path.join(app.config['HUMAN_MODEL_PATH'], "output.obj"))[0] garment_m = glob.glob(os.path.join(app.config['GARMENT_MODEL_PATH'], '*.obj'))[-1] print(f'draping {garment_m} on {human_m}') # Get the file sizes in bytes human_m_size_bytes = os.path.getsize(human_m) garment_m_size_bytes = os.path.getsize(garment_m) # Convert bytes to MB human_m_size_mb = human_m_size_bytes / (1024 * 1024) garment_m_size_mb = garment_m_size_bytes / (1024 * 1024) # Clear existing Blender data bpy.ops.wm.read_factory_settings(use_empty=True) # Import the human model bpy.ops.import_scene.obj(filepath=human_m) human = bpy.context.selected_objects[0] # Import the garment model bpy.ops.import_scene.obj(filepath=garment_m) garment = bpy.context.selected_objects[0] # Align and scale the garment model to the human model human_dimensions = human.dimensions garment_dimensions = garment.dimensions # Scale factor for uniform scaling uniform_scale_factor = min( human_dimensions.x / garment_dimensions.x, human_dimensions.y / garment_dimensions.y, human_dimensions.z / garment_dimensions.z ) + 0.1 garment.scale = (uniform_scale_factor, uniform_scale_factor, uniform_scale_factor) garment.location = human.location garment.location.z += 0.35 # Apply transformations bpy.context.view_layer.objects.active = garment bpy.ops.object.transform_apply(location=True, scale=True, rotation=True) # Add cloth modifier to garment garment.select_set(True) bpy.ops.object.modifier_add(type='CLOTH') # Add collision to human model human.select_set(True) bpy.ops.object.modifier_add(type='COLLISION') # Bake the cloth simulation bpy.context.view_layer.objects.active = garment # bpy.ops.ptcache.bake_all(bake=True) # Export the fitted garment model output_obj_path = os.path.join(app.config['UPLOAD_FOLDER'], 'fitted_model.obj') bpy.ops.export_scene.obj(filepath=output_obj_path, use_selection=True) # Response with file paths and sizes # return jsonify({ # "human_model_path": human_m, # "human_model_size_mb": human_m_size_mb, # "garment_model_path": garment_m, # "garment_model_size_mb": garment_m_size_mb, # "output_obj_path": output_obj_path # }) return send_file(output_obj_path, as_attachment=True) except Exception as e: raise e return jsonify({'error': str(e)}), 500 # @app.route('/') # def index(): # return render_template('index.html') # Route for Human Mesh Generator page @app.route('/') def human(): return render_template('human.html') # Route for Garment Processing page @app.route('/garment') def garment(): return render_template('garment.html') # Route for Draping Garment on Human Model page @app.route('/drape') def drape(): return render_template('drape.html') # Run the Flask app if __name__ == "__main__": app.run(host='0.0.0.0', port=6100)