custom_nodes/dolphin_nodes/dataset_builder/build.py

"""Wan 2.2 LoRA dataset builder.

Feed it a folder of random images/videos; it detects the main subject, crops
to a Wan resolution bucket, captions locally, and writes an
images/ + videos/ + .txt caption dataset plus a starter dataset.toml.

Usage:
    python_embeded\\python.exe dataset_builder\\build.py --input ./raw --output ./dataset_out --trigger "ohwx man"

Run from anywhere -- this script adds its own directory to sys.path so the
sibling modules (crop_resize, detect, caption, scene_split) import cleanly
whether invoked directly or via -m.
"""
import argparse
import os
import sys
import traceback

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

import cv2
import numpy as np
from PIL import Image

import crop_resize
import scene_split
from caption import build_captioner, with_trigger
from detect import detect_subject_bbox, subdivide_bbox

IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".avi", ".webm"}


def parse_args():
    p = argparse.ArgumentParser(description="Wan 2.2 LoRA dataset auto-builder")
    p.add_argument("--input", required=True, help="Folder of raw images/videos (scanned recursively)")
    p.add_argument("--output", required=True, help="Output dataset folder")
    p.add_argument("--trigger", default="", help="Trigger word/phrase prepended to every caption")

    p.add_argument("--captioner", choices=["joycaption", "florence2", "wd14"], default="joycaption")
    p.add_argument("--device", default="cuda")
    p.add_argument("--load-in-4bit", action="store_true", help="4-bit load for JoyCaption (bitsandbytes)")
    p.add_argument("--caption-prompt", default=None, help="Override JoyCaption instruction text")
    p.add_argument("--wd14-general-thresh", type=float, default=0.35)
    p.add_argument("--wd14-character-thresh", type=float, default=0.75)

    p.add_argument("--max-tier", choices=["480p", "720p", "1080p"], default="1080p",
                    help="Highest resolution bucket to consider; auto-picks smaller tiers to avoid upscaling")
    p.add_argument("--face-pad", type=float, default=2.2)
    p.add_argument("--person-pad", type=float, default=1.3)
    p.add_argument("--person-region-pad", type=float, default=1.7,
                    help="padding for person_upper/person_lower crops (see --body-region)")
    p.add_argument("--body-region", choices=["auto", "upper", "lower"], default="auto",
                    help="for person-kind detections, narrow the crop to the top or bottom "
                         "portion of the detected body instead of the whole figure -- use "
                         "when most source frames are already close-ups of one body region. "
                         "'auto' (default) keeps the full detected box. No effect on face-kind "
                         "detections.")
    p.add_argument("--body-region-frac", type=float, default=0.45,
                    help="fraction of the person bbox's height kept when --body-region is upper/lower")
    p.add_argument("--min-face-conf", type=float, default=0.5)
    p.add_argument("--min-person-conf", type=float, default=0.35)
    p.add_argument("--image-format", choices=["jpg", "png"], default="jpg")

    p.add_argument("--video-fps", type=float, default=16.0)
    p.add_argument("--clip-min-sec", type=float, default=2.0)
    p.add_argument("--clip-max-sec", type=float, default=6.0)
    p.add_argument("--scene-diff-thresh", type=float, default=0.45)
    p.add_argument("--clips-per-video", type=int, default=5,
                    help="cap clips extracted per source video, evenly sampled across its "
                         "timeline (e.g. a 5-minute video with many scene cuts won't dump "
                         "dozens of near-duplicate clips into the dataset); pass 0 for no cap")

    return p.parse_args()


def scan_inputs(input_dir):
    images, videos = [], []
    for root, _dirs, files in os.walk(input_dir):
        for name in files:
            ext = os.path.splitext(name)[1].lower()
            path = os.path.join(root, name)
            if ext in IMAGE_EXTS:
                images.append(path)
            elif ext in VIDEO_EXTS:
                videos.append(path)
    return sorted(images), sorted(videos)


def process_image(path, idx, out_dir, args, captioner, pad_overrides):
    pil_img = Image.open(path).convert("RGB")
    bgr = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR)

    bbox, kind = detect_subject_bbox(bgr, args.min_face_conf, args.min_person_conf)
    if kind == "person" and args.body_region != "auto":
        bbox = subdivide_bbox(bbox, args.body_region, args.body_region_frac)
        kind = f"person_{args.body_region}"
    crop_box, (tw, th), bucket_name, tier = crop_resize.compute_crop(
        pil_img.width, pil_img.height, bbox, kind, pad_overrides, args.max_tier
    )
    cropped = crop_resize.crop_resize_pil(pil_img, crop_box, tw, th)

    stem = f"{idx:04d}"
    images_dir = os.path.join(out_dir, "images")
    os.makedirs(images_dir, exist_ok=True)
    img_path = os.path.join(images_dir, f"{stem}.{args.image_format}")
    if args.image_format == "jpg":
        cropped.save(img_path, quality=95)
    else:
        cropped.save(img_path)

    caption = captioner.caption([cropped])
    caption = with_trigger(caption, args.trigger)
    with open(os.path.join(images_dir, f"{stem}.txt"), "w", encoding="utf-8") as f:
        f.write(caption)

    print(f"  [image] {os.path.basename(path)} -> {stem}.{args.image_format} "
          f"({kind}, {bucket_name}/{tier}, {tw}x{th})")


def subsample_ranges(ranges, target):
    """Evenly picks `target` ranges spread across the full list (by index, so
    spread across the video's timeline) instead of just taking the first N --
    keeps whatever diversity of scenes exists rather than favoring the start."""
    if not target or len(ranges) <= target:
        return ranges
    if target <= 1:
        return ranges[len(ranges) // 2 : len(ranges) // 2 + 1]
    step = (len(ranges) - 1) / (target - 1)
    picked_indices = sorted({round(i * step) for i in range(target)})
    return [ranges[i] for i in picked_indices]


def process_video(path, idx_start, out_dir, args, captioner, pad_overrides):
    cuts = scene_split.find_scene_cuts(path, diff_thresh=args.scene_diff_thresh)
    cap = cv2.VideoCapture(path)
    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    src_fps = cap.get(cv2.CAP_PROP_FPS) or args.video_fps
    cap.release()

    ranges = scene_split.build_clip_ranges(
        total_frames, src_fps, cuts, args.clip_min_sec, args.clip_max_sec
    )
    detected_count = len(ranges)
    ranges = subsample_ranges(ranges, args.clips_per_video)
    if args.clips_per_video and detected_count > len(ranges):
        print(f"  [video] {os.path.basename(path)}: {detected_count} scene(s) detected, "
              f"sampling {len(ranges)} (--clips-per-video {args.clips_per_video})")

    idx = idx_start
    for start, end in ranges:
        result = scene_split.extract_clip(
            path, start, end, args.video_fps, args.max_tier, pad_overrides,
            args.body_region, args.body_region_frac,
        )
        if result is None:
            continue
        frames, (tw, th), representative, motion_tag = result

        stem = f"{idx:04d}"
        videos_dir = os.path.join(out_dir, "videos")
        os.makedirs(videos_dir, exist_ok=True)
        clip_path = os.path.join(videos_dir, f"{stem}.mp4")
        scene_split.write_clip_mp4(frames, clip_path, args.video_fps)

        caption = captioner.caption([representative])
        caption = f"{caption.strip()}, {motion_tag}" if caption.strip() else motion_tag
        caption = with_trigger(caption, args.trigger)
        with open(os.path.join(videos_dir, f"{stem}.txt"), "w", encoding="utf-8") as f:
            f.write(caption)

        print(f"  [video] {os.path.basename(path)} [{start}-{end}] -> {stem}.mp4 "
              f"({len(frames)}f @ {args.video_fps}fps, {tw}x{th}, {motion_tag})")
        idx += 1
    return idx


def write_dataset_toml(out_dir, args, has_images, has_videos):
    """musubi-tuner dataset config (schema verified against docs/dataset_config.md).
    Items are already pre-cropped to one of a handful of Wan resolution buckets
    by this tool, so enable_bucket lets musubi-tuner group them without forcing
    a single resolution; `resolution` below is just the nominal/reference size."""
    ref_w, ref_h = crop_resize.bucket_set(args.max_tier)["landscape"]

    def p(*parts):
        return os.path.join(out_dir, *parts).replace("\\", "/")

    lines = [
        "[general]",
        f"resolution = [{ref_w}, {ref_h}]",
        'caption_extension = ".txt"',
        "batch_size = 1",
        "enable_bucket = true",
        "bucket_no_upscale = true",
        "",
    ]
    if has_images:
        lines += [
            "[[datasets]]",
            f'image_directory = "{p("images")}"',
            f'cache_directory = "{p("images", "cache")}"',
            "num_repeats = 1",
            "",
        ]
    if has_videos:
        max_frames = int(args.video_fps * args.clip_max_sec) + 1
        lines += [
            "[[datasets]]",
            f'video_directory = "{p("videos")}"',
            f'cache_directory = "{p("videos", "cache")}"',
            'frame_extraction = "full"',
            f"max_frames = {max_frames}",
            f"source_fps = {args.video_fps}",
            "num_repeats = 1",
            "",
        ]
    with open(os.path.join(out_dir, "dataset.toml"), "w", encoding="utf-8") as f:
        f.write("\n".join(lines))


def main():
    args = parse_args()
    os.makedirs(os.path.join(args.output, "images"), exist_ok=True)
    os.makedirs(os.path.join(args.output, "videos"), exist_ok=True)

    images, videos = scan_inputs(args.input)
    print(f"Found {len(images)} image(s), {len(videos)} video(s) under {args.input}")
    if not images and not videos:
        print("Nothing to do.")
        return

    pad_overrides = {
        "face": args.face_pad, "person": args.person_pad,
        "person_upper": args.person_region_pad, "person_lower": args.person_region_pad,
    }

    print(f"Loading captioner: {args.captioner} ...")
    captioner = build_captioner(args)

    img_idx = 1
    for path in images:
        try:
            process_image(path, img_idx, args.output, args, captioner, pad_overrides)
            img_idx += 1
        except Exception:
            print(f"  [image] FAILED {path}:")
            traceback.print_exc()

    vid_idx = 1
    for path in videos:
        try:
            vid_idx = process_video(path, vid_idx, args.output, args, captioner, pad_overrides)
        except Exception:
            print(f"  [video] FAILED {path}:")
            traceback.print_exc()

    write_dataset_toml(args.output, args, bool(images), bool(videos))
    print(f"Done. {img_idx - 1} image(s), {vid_idx - 1} clip(s) -> {args.output}")


if __name__ == "__main__":
    main()