Download scripts/data/patch_2264_source_motion_dir.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/patch_2264_source_motion_dir.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/patch_2264_source_motion_dir.py
-
curl -L -o patch_2264_source_motion_dir.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/patch_2264_source_motion_dir.py
13.9 kB
| #!/usr/bin/env python3 | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import multiprocessing as mp | |
| import os | |
| import re | |
| import shutil | |
| import sys | |
| import time | |
| import traceback | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Any | |
| NAME_RE = re.compile(rb'"name"\s*:\s*"([^"]+)"') | |
| SOURCE_MOTION_RE = re.compile(rb'"source_motion_(?:dir|path)"\s*:') | |
| SOURCE_VIDEO_RE = re.compile(rb'"source_video_path"\s*:') | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser( | |
| description=( | |
| "Patch 2264_with_audio video_clip metadata.jsonl with source_motion_dir " | |
| "so it matches existing metadata_only video_clip datasets." | |
| ) | |
| ) | |
| parser.add_argument( | |
| "--clip-root", | |
| type=Path, | |
| default=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full/2264_with_audio"), | |
| ) | |
| parser.add_argument( | |
| "--source-root", | |
| type=Path, | |
| default=Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio"), | |
| ) | |
| parser.add_argument("--metadata", default="metadata.jsonl") | |
| parser.add_argument("--source-metadata", default="metadata_convert.jsonl") | |
| parser.add_argument("--workers", type=int, default=min(16, os.cpu_count() or 1)) | |
| parser.add_argument("--progress-bytes", type=int, default=64 * 1024 * 1024) | |
| parser.add_argument("--apply", action="store_true", help="Atomically replace metadata.jsonl after successful patch.") | |
| parser.add_argument("--keep-temp", action="store_true") | |
| parser.add_argument("--no-space-check", action="store_true") | |
| return parser.parse_args() | |
| def load_source_index(source_root: Path, source_metadata: Path) -> dict[str, tuple[bytes, bytes]]: | |
| if not source_metadata.exists(): | |
| raise FileNotFoundError(f"source metadata not found: {source_metadata}") | |
| index: dict[str, tuple[bytes, bytes]] = {} | |
| missing: list[str] = [] | |
| with source_metadata.open("r", encoding="utf-8") as f: | |
| for line_no, line in enumerate(f, start=1): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| row = json.loads(line) | |
| name = str(row.get("name", "")).strip() | |
| motion_rel = row.get("converted_path") | |
| video_rel = row.get("video_path") | |
| if not name or not motion_rel or not video_rel: | |
| missing.append(f"{line_no}: missing name/converted_path/video_path") | |
| continue | |
| motion_path = source_root / str(motion_rel) | |
| video_path = source_root / str(video_rel) | |
| if not (motion_path / "joint_pos.csv").exists(): | |
| missing.append(f"{line_no}: missing motion csv dir {motion_path}") | |
| continue | |
| if not video_path.exists(): | |
| missing.append(f"{line_no}: missing source video {video_path}") | |
| continue | |
| motion_json = json.dumps(str(motion_path), ensure_ascii=False, separators=(",", ":")).encode("utf-8") | |
| video_json = json.dumps(str(video_path), ensure_ascii=False, separators=(",", ":")).encode("utf-8") | |
| index[name] = (motion_json, video_json) | |
| if missing: | |
| preview = "\n".join(missing[:20]) | |
| raise RuntimeError(f"source metadata validation failed ({len(missing)} issue(s)); first issues:\n{preview}") | |
| if not index: | |
| raise RuntimeError(f"empty source index from {source_metadata}") | |
| return index | |
| def compute_chunks(path: Path, workers: int) -> list[tuple[int, int, int]]: | |
| size = path.stat().st_size | |
| workers = max(1, min(int(workers), size or 1)) | |
| starts = [0] | |
| with path.open("rb") as f: | |
| for i in range(1, workers): | |
| f.seek(size * i // workers) | |
| f.readline() | |
| pos = f.tell() | |
| if pos < size and pos != starts[-1]: | |
| starts.append(pos) | |
| starts.append(size) | |
| return [(idx, starts[idx], starts[idx + 1]) for idx in range(len(starts) - 1) if starts[idx] < starts[idx + 1]] | |
| def patch_line(line: bytes, source_index: dict[str, tuple[bytes, bytes]]) -> tuple[bytes, str | None]: | |
| match = NAME_RE.search(line) | |
| if match is None: | |
| return line, "missing name" | |
| clip_name = match.group(1).decode("utf-8") | |
| source_name = clip_name.split(".chunk", 1)[0] | |
| source = source_index.get(source_name) | |
| if source is None: | |
| return line, f"missing source mapping for {source_name}" | |
| additions: list[bytes] = [] | |
| if SOURCE_MOTION_RE.search(line) is None: | |
| additions.append(b'"source_motion_dir":' + source[0]) | |
| if SOURCE_VIDEO_RE.search(line) is None: | |
| additions.append(b'"source_video_path":' + source[1]) | |
| if not additions: | |
| return line, None | |
| newline = b"\n" if line.endswith(b"\n") else b"" | |
| body = line.rstrip(b"\r\n") | |
| if not body.endswith(b"}"): | |
| return line, "line is not a JSON object" | |
| return body[:-1] + b"," + b",".join(additions) + b"}" + newline, None | |
| def worker_main( | |
| idx: int, | |
| start: int, | |
| end: int, | |
| input_path: str, | |
| output_path: str, | |
| source_index: dict[str, tuple[bytes, bytes]], | |
| queue: mp.Queue, | |
| progress_bytes: int, | |
| ) -> None: | |
| stats: dict[str, Any] = { | |
| "idx": idx, | |
| "rows": 0, | |
| "patched": 0, | |
| "already_had_source_motion": 0, | |
| "already_had_source_video": 0, | |
| "errors": 0, | |
| "error_examples": [], | |
| "bytes": 0, | |
| } | |
| try: | |
| next_report = progress_bytes | |
| with open(input_path, "rb") as fin, open(output_path, "wb") as fout: | |
| fin.seek(start) | |
| while fin.tell() < end: | |
| line = fin.readline() | |
| if not line: | |
| break | |
| stats["rows"] += 1 | |
| stats["bytes"] += len(line) | |
| had_motion = SOURCE_MOTION_RE.search(line) is not None | |
| had_video = SOURCE_VIDEO_RE.search(line) is not None | |
| if had_motion: | |
| stats["already_had_source_motion"] += 1 | |
| if had_video: | |
| stats["already_had_source_video"] += 1 | |
| patched, error = patch_line(line, source_index) | |
| if error is not None: | |
| stats["errors"] += 1 | |
| if len(stats["error_examples"]) < 10: | |
| stats["error_examples"].append(f"chunk={idx} row={stats['rows']}: {error}") | |
| fout.write(line) | |
| else: | |
| if patched != line: | |
| stats["patched"] += 1 | |
| fout.write(patched) | |
| if stats["bytes"] >= next_report: | |
| queue.put(("progress", idx, stats["bytes"])) | |
| next_report = stats["bytes"] + progress_bytes | |
| queue.put(("progress", idx, stats["bytes"])) | |
| queue.put(("done", idx, stats)) | |
| except BaseException: | |
| queue.put(("error", idx, traceback.format_exc())) | |
| def progress_bar(total: int): | |
| try: | |
| from tqdm import tqdm | |
| return tqdm(total=total, unit="B", unit_scale=True, desc="patch metadata") | |
| except Exception: | |
| return None | |
| def run_workers( | |
| metadata_path: Path, | |
| tmp_dir: Path, | |
| source_index: dict[str, tuple[bytes, bytes]], | |
| workers: int, | |
| progress_bytes: int, | |
| ) -> tuple[list[Path], dict[str, Any]]: | |
| chunks = compute_chunks(metadata_path, workers) | |
| queue: mp.Queue = mp.Queue() | |
| processes: list[mp.Process] = [] | |
| part_paths: list[Path] = [] | |
| for idx, start, end in chunks: | |
| part_path = tmp_dir / f"metadata.part-{idx:05d}.jsonl" | |
| part_paths.append(part_path) | |
| process = mp.Process( | |
| target=worker_main, | |
| args=(idx, start, end, str(metadata_path), str(part_path), source_index, queue, progress_bytes), | |
| ) | |
| process.start() | |
| processes.append(process) | |
| pbar = progress_bar(metadata_path.stat().st_size) | |
| last_progress = {idx: 0 for idx, _, _ in chunks} | |
| totals: dict[str, Any] = { | |
| "rows": 0, | |
| "patched": 0, | |
| "already_had_source_motion": 0, | |
| "already_had_source_video": 0, | |
| "errors": 0, | |
| "error_examples": [], | |
| } | |
| done = 0 | |
| last_print = time.time() | |
| try: | |
| while done < len(processes): | |
| kind, idx, payload = queue.get() | |
| if kind == "progress": | |
| delta = int(payload) - last_progress.get(idx, 0) | |
| last_progress[idx] = int(payload) | |
| if pbar is not None: | |
| pbar.update(max(0, delta)) | |
| elif time.time() - last_print > 5: | |
| seen = sum(last_progress.values()) | |
| print(f"[patch] {seen / (1024 ** 3):.2f}/{metadata_path.stat().st_size / (1024 ** 3):.2f} GB", flush=True) | |
| last_print = time.time() | |
| elif kind == "done": | |
| done += 1 | |
| for key in ("rows", "patched", "already_had_source_motion", "already_had_source_video", "errors"): | |
| totals[key] += int(payload[key]) | |
| totals["error_examples"].extend(payload["error_examples"]) | |
| elif kind == "error": | |
| for process in processes: | |
| if process.is_alive(): | |
| process.terminate() | |
| raise RuntimeError(f"worker {idx} failed:\n{payload}") | |
| finally: | |
| if pbar is not None: | |
| pbar.close() | |
| for process in processes: | |
| process.join() | |
| return part_paths, totals | |
| def merge_parts(part_paths: list[Path], output_path: Path) -> None: | |
| try: | |
| from tqdm import tqdm | |
| pbar = tqdm(total=sum(path.stat().st_size for path in part_paths), unit="B", unit_scale=True, desc="merge") | |
| except Exception: | |
| pbar = None | |
| with output_path.open("wb") as fout: | |
| for part in part_paths: | |
| with part.open("rb") as fin: | |
| while True: | |
| chunk = fin.read(64 * 1024 * 1024) | |
| if not chunk: | |
| break | |
| fout.write(chunk) | |
| if pbar is not None: | |
| pbar.update(len(chunk)) | |
| if pbar is not None: | |
| pbar.close() | |
| def check_free_space(root: Path, metadata_path: Path) -> None: | |
| usage = shutil.disk_usage(root) | |
| required = int(metadata_path.stat().st_size * 1.35) + 8 * 1024**3 | |
| if usage.free < required: | |
| raise RuntimeError( | |
| f"not enough free space under {root}: free={usage.free / (1024 ** 3):.1f}GB, " | |
| f"need about {required / (1024 ** 3):.1f}GB" | |
| ) | |
| def main() -> None: | |
| args = parse_args() | |
| clip_root: Path = args.clip_root | |
| source_root: Path = args.source_root | |
| metadata_path = clip_root / args.metadata | |
| source_metadata = source_root / args.source_metadata | |
| if not metadata_path.exists(): | |
| raise FileNotFoundError(f"clip metadata not found: {metadata_path}") | |
| if not args.no_space_check: | |
| check_free_space(clip_root, metadata_path) | |
| print(f"[source] loading {source_metadata}", flush=True) | |
| source_index = load_source_index(source_root, source_metadata) | |
| print(f"[source] valid source motions/videos: {len(source_index)}", flush=True) | |
| timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") | |
| tmp_dir = clip_root / f".source_motion_patch_tmp.{timestamp}.{os.getpid()}" | |
| tmp_dir.mkdir(parents=False, exist_ok=False) | |
| try: | |
| part_paths, totals = run_workers( | |
| metadata_path=metadata_path, | |
| tmp_dir=tmp_dir, | |
| source_index=source_index, | |
| workers=args.workers, | |
| progress_bytes=args.progress_bytes, | |
| ) | |
| if totals["errors"]: | |
| preview = "\n".join(totals["error_examples"][:20]) | |
| raise RuntimeError(f"patch produced {totals['errors']} error(s); first issues:\n{preview}") | |
| patched_metadata = tmp_dir / "metadata.jsonl" | |
| merge_parts(part_paths, patched_metadata) | |
| new_lines = sum(1 for _ in patched_metadata.open("rb")) | |
| if new_lines != totals["rows"]: | |
| raise RuntimeError(f"line count mismatch after merge: worker_rows={totals['rows']} merged_lines={new_lines}") | |
| print( | |
| "[summary] " | |
| f"rows={totals['rows']} patched={totals['patched']} " | |
| f"already_source_motion={totals['already_had_source_motion']} " | |
| f"already_source_video={totals['already_had_source_video']} " | |
| f"output={patched_metadata}", | |
| flush=True, | |
| ) | |
| if not args.apply: | |
| print("[dry-run] not replacing metadata.jsonl. Re-run with --apply to install the patched file.", flush=True) | |
| return | |
| backup_path = clip_root / f"{args.metadata}.before_source_motion_patch.{timestamp}" | |
| print(f"[apply] backup original -> {backup_path}", flush=True) | |
| os.replace(metadata_path, backup_path) | |
| try: | |
| os.replace(patched_metadata, metadata_path) | |
| except BaseException: | |
| if metadata_path.exists(): | |
| failed_path = clip_root / f"{args.metadata}.failed_source_motion_patch.{timestamp}" | |
| os.replace(metadata_path, failed_path) | |
| os.replace(backup_path, metadata_path) | |
| raise | |
| print(f"[apply] installed patched metadata: {metadata_path}", flush=True) | |
| print(f"[apply] backup kept at: {backup_path}", flush=True) | |
| finally: | |
| if not args.keep_temp and tmp_dir.exists(): | |
| shutil.rmtree(tmp_dir, ignore_errors=True) | |
| if __name__ == "__main__": | |
| try: | |
| main() | |
| except KeyboardInterrupt: | |
| print("interrupted", file=sys.stderr) | |
| raise | |