boom-vvt / src /process /dwpose.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
6ba0ed9 verified
Raw History Blame Contribute Delete
8.69 kB
import argparse
import colorsys
import json
import math
import os
import cv2
import numpy as np
from PIL import Image
from rtmlib import Wholebody
from diffusers.utils import export_to_video, load_video
BODY_KPTS = 18
HAND_KPTS = 21
FACE_START, FACE_END = 24, 92
LEFT_HAND_START, LEFT_HAND_END = 92, 113
RIGHT_HAND_START, RIGHT_HAND_END = 113, 134
def flatten_xy_score(xy, score, num_points):
values = []
for i in range(num_points):
if i < len(xy):
values.extend([float(xy[i, 0]), float(xy[i, 1]), float(score[i])])
else:
values.extend([0.0, 0.0, 0.0])
return values
def person_to_openpose(xy, score, include_hands=True):
person = {
"pose_keypoints_2d": flatten_xy_score(xy[:BODY_KPTS], score[:BODY_KPTS], BODY_KPTS),
"face_keypoints_2d": [],
}
if include_hands and len(xy) >= RIGHT_HAND_END:
person["hand_left_keypoints_2d"] = flatten_xy_score(
xy[LEFT_HAND_START:LEFT_HAND_END], score[LEFT_HAND_START:LEFT_HAND_END], HAND_KPTS
)
person["hand_right_keypoints_2d"] = flatten_xy_score(
xy[RIGHT_HAND_START:RIGHT_HAND_END], score[RIGHT_HAND_START:RIGHT_HAND_END], HAND_KPTS
)
else:
person["hand_left_keypoints_2d"] = []
person["hand_right_keypoints_2d"] = []
return person
def frame_to_openpose(keypoints, scores, width, height, include_hands=True):
people = []
if keypoints is not None and len(keypoints) > 0:
order = np.argsort(scores[:, :BODY_KPTS].sum(axis=1))[::-1]
for idx in order:
people.append(person_to_openpose(keypoints[idx], scores[idx], include_hands=include_hands))
return {
"people": people,
"canvas_width": int(width),
"canvas_height": int(height),
}
OPENPOSE_LIMBS = [
(1, 2), (1, 5), (2, 3), (3, 4), (5, 6), (6, 7), (1, 8), (8, 9), (9, 10),
(1, 11), (11, 12), (12, 13), (1, 0), (0, 14), (14, 16), (0, 15), (15, 17),
]
OPENPOSE_COLORS = [
[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0],
[85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255],
[0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255],
[255, 0, 255], [255, 0, 170], [255, 0, 85],
]
HAND_EDGES = [
(0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8), (0, 9),
(9, 10), (10, 11), (11, 12), (0, 13), (13, 14), (14, 15), (15, 16),
(0, 17), (17, 18), (18, 19), (19, 20),
]
def draw_bodypose(canvas, keypoints, scores, kpt_thr=0.3):
stickwidth = 4
for i, (src, dst) in enumerate(OPENPOSE_LIMBS):
if scores[src] < kpt_thr or scores[dst] < kpt_thr:
continue
x0, y0 = keypoints[src]
x1, y1 = keypoints[dst]
mx = 0.5 * (x0 + x1)
my = 0.5 * (y0 + y1)
length = math.hypot(x0 - x1, y0 - y1)
if length < 1:
continue
angle = math.degrees(math.atan2(y0 - y1, x0 - x1))
polygon = cv2.ellipse2Poly(
(int(mx), int(my)), (int(length / 2), stickwidth), int(angle), 0, 360, 1
)
overlay = canvas.copy()
cv2.fillConvexPoly(overlay, polygon, OPENPOSE_COLORS[i])
cv2.addWeighted(canvas, 0.4, overlay, 0.6, 0, dst=canvas)
for i in range(min(BODY_KPTS, len(keypoints))):
if scores[i] < kpt_thr:
continue
x, y = int(keypoints[i, 0]), int(keypoints[i, 1])
cv2.circle(canvas, (x, y), 4, OPENPOSE_COLORS[i], thickness=-1)
return canvas
def draw_handpose(canvas, keypoints, scores, kpt_thr=0.3):
for ie, (src, dst) in enumerate(HAND_EDGES):
if scores[src] < kpt_thr or scores[dst] < kpt_thr:
continue
x1, y1 = keypoints[src]
x2, y2 = keypoints[dst]
if min(x1, y1, x2, y2) <= 1:
continue
color = [int(c * 255) for c in colorsys.hsv_to_rgb(ie / float(len(HAND_EDGES)), 1.0, 1.0)]
cv2.line(canvas, (int(x1), int(y1)), (int(x2), int(y2)), color, thickness=2)
for i in range(min(HAND_KPTS, len(keypoints))):
if scores[i] < kpt_thr:
continue
x, y = keypoints[i]
if x <= 1 and y <= 1:
continue
cv2.circle(canvas, (int(x), int(y)), 4, (0, 0, 255), thickness=-1)
return canvas
def render_pose(height, width, keypoints, scores, kpt_thr=0.3, include_hands=True):
canvas = np.zeros((height, width, 3), dtype=np.uint8)
if keypoints is None or len(keypoints) == 0:
return Image.fromarray(canvas)
for person_kpts, person_scores in zip(keypoints, scores):
draw_bodypose(canvas, person_kpts[:BODY_KPTS], person_scores[:BODY_KPTS], kpt_thr=kpt_thr)
if include_hands and len(person_kpts) >= RIGHT_HAND_END:
draw_handpose(
canvas,
person_kpts[LEFT_HAND_START:LEFT_HAND_END],
person_scores[LEFT_HAND_START:LEFT_HAND_END],
kpt_thr=kpt_thr,
)
draw_handpose(
canvas,
person_kpts[RIGHT_HAND_START:RIGHT_HAND_END],
person_scores[RIGHT_HAND_START:RIGHT_HAND_END],
kpt_thr=kpt_thr,
)
return Image.fromarray(canvas)
def resolve_device(device):
requested = (device or "cpu").strip()
if requested == "cpu":
return "cpu"
try:
import onnxruntime as ort
providers = ort.get_available_providers()
except Exception:
providers = []
if "CUDAExecutionProvider" not in providers:
print(f"[dwpose] {requested} 不可用(当前 ONNX Runtime 无 CUDA),改用 cpu")
return "cpu"
return requested
def init_model(mode="balanced", device="cuda", backend="onnxruntime"):
device = resolve_device(device)
print(f"[dwpose] init model: mode={mode}, device={device}, backend={backend}")
return Wholebody(
to_openpose=True,
mode=mode,
backend=backend,
device=device,
)
def data_process(video_path, num_frames=65):
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise RuntimeError(f"无法打开视频: {video_path}")
fps = cap.get(cv2.CAP_PROP_FPS)
cap.release()
frames = load_video(video_path)
if not frames:
raise ValueError(f"视频为空: {video_path}")
if num_frames is not None and int(num_frames) >= 0:
frames = frames[: int(num_frames)]
print(f"[data] frames={len(frames)}, size={frames[0].size}, fps={fps}")
return frames, fps
def infer_dwpose(estimator, frames, include_hands=True, kpt_thr=0.3):
pose_json = []
pose_video = []
for frame_idx, frame in enumerate(frames):
width, height = frame.size
image = np.array(frame.convert("RGB"))[:, :, ::-1]
keypoints, scores = estimator(image)
pose_json.append(frame_to_openpose(keypoints, scores, width, height, include_hands=include_hands))
pose_video.append(
render_pose(height, width, keypoints, scores, kpt_thr=kpt_thr, include_hands=include_hands)
)
if (frame_idx + 1) % 20 == 0:
print(f"[dwpose] frame={frame_idx + 1}/{len(frames)}")
return pose_json, pose_video
def parse_args():
parser = argparse.ArgumentParser(description="DWPose pose detection")
parser.add_argument("--video", required=True, help="person video path")
parser.add_argument("--output", required=True, help="output directory")
parser.add_argument("--device", default="cuda")
parser.add_argument("--mode", default="balanced", choices=["lightweight", "balanced", "performance"])
parser.add_argument("--backend", default="onnxruntime")
parser.add_argument("--kpt_thr", type=float, default=0.3)
parser.add_argument("--include_hands", action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--num_frames", type=int, default=65, help="max frames to process, -1 for all")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
os.makedirs(args.output, exist_ok=True)
save_json = os.path.join(args.output, "dwpose.json")
save_video = os.path.join(args.output, "dwpose.mp4")
estimator = init_model(mode=args.mode, device=args.device, backend=args.backend)
frames, fps = data_process(args.video, num_frames=args.num_frames)
pose_json, pose_video = infer_dwpose(
estimator, frames, include_hands=args.include_hands, kpt_thr=args.kpt_thr
)
with open(save_json, "w", encoding="utf-8") as f:
json.dump(pose_json, f)
export_to_video(pose_video, save_video, fps=fps)
print(f"[dwpose] json={save_json}")
print(f"[dwpose] video={save_video}")