Download scripts/data/package_hf_audio_motion.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 20 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/package_hf_audio_motion.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/package_hf_audio_motion.py
-
curl -L -o package_hf_audio_motion.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/package_hf_audio_motion.py
20 kB
| #!/usr/bin/env python3 | |
| """Offline HF handoff packaging. Default is a read-only plan; never uploads.""" | |
| from __future__ import annotations | |
| import argparse | |
| import concurrent.futures | |
| import fcntl | |
| import hashlib | |
| import io | |
| import json | |
| import os | |
| from pathlib import Path | |
| import re | |
| import shutil | |
| import sqlite3 | |
| import subprocess | |
| import tarfile | |
| import time | |
| ROOT = Path(__file__).resolve().parents[2] | |
| PACKAGES = ( | |
| "hfd_audio_command_continuous_office_v1", | |
| "hfd_control_commands_v1", | |
| "hfd_backchannel_commands_v2_short", | |
| "hfd_hesitation_pause_v1", | |
| "hfd_hesitation_pause_v2", | |
| ) | |
| THIRD_PARTY_PACKAGE = "hfd_third_party_multi_v5" | |
| SKIP_DIRS = {".git", ".venv-psi", "__pycache__", ".pytest_cache", ".cache", | |
| ".progress", "logs", "gt_review", "runs", "checkpoints"} | |
| CODE_DIRS = ("src", "scripts", "tests", "assets", "backup") | |
| CODE_FILES = ("README.md", "pyproject.toml", "uv.lock", "posttrain-full-duplex-audio-command.sh", | |
| "infer_render_auto.sh") | |
| def files_under(path): | |
| """Never follow a link into another dataset, environment or private tree.""" | |
| path = Path(path) | |
| if path.is_symlink(): | |
| raise ValueError(f"Refusing symlink: {path}") | |
| if path.is_file(): | |
| yield path | |
| return | |
| if not path.is_dir(): | |
| raise FileNotFoundError(path) | |
| for directory, dirs, names in os.walk(path, followlinks=False): | |
| dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) | |
| for name in [*dirs, *names]: | |
| item = Path(directory) / name | |
| if item.is_symlink(): | |
| raise ValueError(f"Refusing symlink: {item}") | |
| for name in sorted(names): | |
| if (name.startswith(".env") or name in {"id_rsa", "id_ed25519"} | |
| or name.endswith((".pyc", ".pyo", ".key", ".pem"))): | |
| continue | |
| yield Path(directory) / name | |
| def code_entries(root): | |
| roots = [root / p for p in (*CODE_DIRS, *CODE_FILES)] | |
| roots.append(root / "doc/hf_audio_motion") | |
| for base in roots: | |
| for path in files_under(base): | |
| if path.suffix.lower() in {'.pt', '.pth', '.safetensors', '.wav', '.mp4', '.pkl', '.sqlite', '.bin'}: | |
| continue | |
| yield path, "HFD/" + path.relative_to(root).as_posix() | |
| def digest(path): | |
| h = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for block in iter(lambda: handle.read(8 * 1024 * 1024), b""): | |
| h.update(block) | |
| return h.hexdigest() | |
| def safe_code_bytes(path): | |
| """Remove historical commented tokens in export only; reject executable secrets.""" | |
| value = path.read_bytes() | |
| if re.search(rb'-----BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY-----', value): | |
| raise ValueError(f'Private key detected; not exporting {path}') | |
| token = re.compile(rb'(?<![A-Za-z0-9_])(?:hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9_-]{20,})') | |
| lines = [] | |
| for line in value.splitlines(keepends=True): | |
| if token.search(line): | |
| if not line.lstrip().startswith(b'#'): | |
| raise ValueError(f'Possible executable credential; review {path}') | |
| line = token.sub(b'[REDACTED_TOKEN]', line) | |
| lines.append(line) | |
| return b''.join(lines) | |
| def write_archive(target, entries, extras=None, resume_partial=False): | |
| """Atomic archive; no destructive replacement of an earlier export.""" | |
| if target.exists(): | |
| raise FileExistsError(target) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = target.with_name(target.name + ".partial") | |
| if temporary.exists() and not resume_partial: | |
| raise FileExistsError(temporary) | |
| count = raw_bytes = 0 | |
| with tarfile.open(temporary, "w:gz", compresslevel=1, dereference=False) as archive: | |
| for path, name in entries: | |
| if path.is_symlink() or not path.is_file(): | |
| raise ValueError(f"Not a regular file: {path}") | |
| before = path.stat() | |
| value = safe_code_bytes(path) | |
| info = archive.gettarinfo(str(path), arcname=name) | |
| info.size = len(value) | |
| archive.addfile(info, io.BytesIO(value)) | |
| after = path.stat() | |
| if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): | |
| raise RuntimeError(f"Source changed during packaging: {path}") | |
| count += 1 | |
| raw_bytes += len(value) | |
| if count % 20000 == 0: | |
| print(f"[pack] {target.name}: {count:,} files", flush=True) | |
| for name, value in (extras or {}).items(): | |
| info = tarfile.TarInfo(name) | |
| info.size = len(value) | |
| info.mode = 0o644 | |
| archive.addfile(info, io.BytesIO(value)) | |
| count += 1 | |
| raw_bytes += len(value) | |
| os.replace(temporary, target) | |
| result = {"file": target.name, "files": count, "raw_bytes": raw_bytes, | |
| "archive_bytes": target.stat().st_size, "sha256": digest(target)} | |
| print(json.dumps(result), flush=True) | |
| return result | |
| def dataset_jobs(motion, tts, data, third_party=None): | |
| # Preserve source bytes/IDs. Absolute paths are handled by the receiving loader. | |
| jobs = [("seed_motion", [(motion / "g1/csv_sonic", "motion/seed_all/g1/csv_sonic"), | |
| (motion / "metadata/seed_metadata_v004.csv", "motion/seed_all/metadata/seed_metadata_v004.csv")]), | |
| ("seed_tts", [(tts / "seed_audio", "audio/seed_tts/seed_audio"), | |
| (tts / "metadata.jsonl", "audio/seed_tts/metadata.jsonl")])] | |
| jobs += [(name, [(data / name, "audio/" + name)]) for name in PACKAGES] | |
| if third_party is not None: | |
| jobs.append((THIRD_PARTY_PACKAGE, [ | |
| (third_party / "production", "audio/hfd_third_party_multi_v5/production"), | |
| (third_party / "training_scale_all_commands", | |
| "audio/hfd_third_party_multi_v5/training_scale_all_commands"), | |
| ])) | |
| return jobs | |
| def mapped_entries(sources): | |
| for source, destination in sources: | |
| for path in files_under(source): | |
| name = destination if source.is_file() else destination + "/" + path.relative_to(source).as_posix() | |
| yield path, name | |
| def plan_shards(name, sources, count): | |
| """One motion folder per sample; other files are independent samples. | |
| Stable hash assignment avoids depending on filesystem traversal order. | |
| Metadata JSONL/CSV remains byte-identical and is addressable via SQLite. | |
| """ | |
| buckets = [dict() for _ in range(count)] | |
| total = 0 | |
| for path, logical in mapped_entries(sources): | |
| sample_path = logical.rsplit('/', 1)[0] if name == 'seed_motion' and '/g1/csv_sonic/' in logical else logical | |
| key = hashlib.sha256(sample_path.encode()).hexdigest()[:40] | |
| bucket = buckets[int(key[:16], 16) % count] | |
| field = path.name.lower() if sample_path != logical else 'payload' + (path.suffix.lower() or '.bin') | |
| if any(ch not in 'abcdefghijklmnopqrstuvwxyz0123456789_.' for ch in field): | |
| field = 'payload.bin' | |
| st = path.stat() | |
| bucket.setdefault(key, []).append((path, logical, field, st.st_size, st.st_mtime_ns)) | |
| total += st.st_size | |
| return buckets, total | |
| def pack_shard(dest, name, number, samples): | |
| """Uncompressed, contiguous WDS samples + offsets. Resume verified shards.""" | |
| rel = f'shards/{name}/{number:05d}.tar' | |
| target = dest / rel | |
| journal = dest / '.build' / name / f'{number:05d}.jsonl' | |
| marker = journal.with_suffix('.done.json') | |
| inventory = [(k, str(p), n, f, size, mt) for k, rows in sorted(samples.items()) for p, n, f, size, mt in rows] | |
| signature = hashlib.sha256(json.dumps(inventory, ensure_ascii=False).encode()).hexdigest() | |
| if marker.exists(): | |
| record = json.loads(marker.read_text()) | |
| if record['source_signature'] != signature: | |
| raise RuntimeError(f'Sources changed: {rel}; choose a NEW export directory') | |
| if not target.exists() or not journal.exists() or digest(target) != record['sha256'] or digest(journal) != record['index_sha256']: | |
| raise RuntimeError(f'Completed shard corrupted: {rel}') | |
| print(f'[resume] {rel}', flush=True) | |
| return record | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| journal.parent.mkdir(parents=True, exist_ok=True) | |
| partial = target.with_suffix('.tar.partial') | |
| count = raw = 0 | |
| with tarfile.open(partial, 'w', format=tarfile.USTAR_FORMAT) as tar, journal.open('w') as index: | |
| for key, rows in sorted(samples.items()): | |
| fields = [row[2] for row in rows] | |
| if len(fields) != len(set(fields)) or 'json' in fields: | |
| raise ValueError(f'Duplicate WDS fields: {key}') | |
| meta = {'key': key, 'package': name, 'paths': {row[2]: row[1] for row in rows}} | |
| content = json.dumps(meta, ensure_ascii=False).encode() | |
| info = tarfile.TarInfo(key + '.json') | |
| info.size = len(content) | |
| tar.addfile(info, io.BytesIO(content)) | |
| for path, logical, field, size, mtime in rows: | |
| before = path.stat() | |
| if path.is_symlink() or (before.st_size, before.st_mtime_ns) != (size, mtime): | |
| raise RuntimeError(f'Source changed since planning: {path}') | |
| info = tarfile.TarInfo(key + '.' + field) | |
| info.size = size | |
| offset = tar.offset + 512 # USTAR, short ASCII names: exactly one header block. | |
| with path.open('rb') as handle: | |
| tar.addfile(info, handle) | |
| after = path.stat() | |
| if (after.st_size, after.st_mtime_ns) != (size, mtime): | |
| raise RuntimeError(f'Source changed during packaging: {path}') | |
| index.write(json.dumps([logical, rel, offset, size, key, field], ensure_ascii=False) + '\n') | |
| count += 1 | |
| raw += size | |
| os.replace(partial, target) | |
| record = {'file': rel, 'files': count, 'samples': len(samples), 'raw_bytes': raw, | |
| 'archive_bytes': target.stat().st_size, 'sha256': digest(target), | |
| 'source_signature': signature, 'index_sha256': digest(journal)} | |
| tmp = marker.with_suffix('.partial') | |
| tmp.write_text(json.dumps(record) + '\n') | |
| os.replace(tmp, marker) # Only this marker constitutes a completed shard. | |
| print(f'[done] {rel}: {len(samples):,} samples, {raw / 2**20:.1f} MiB', flush=True) | |
| return record | |
| def verify_package_plan(dest, name, buckets): | |
| """Also detect removals that would make a previously completed shard empty.""" | |
| hasher = hashlib.sha256() | |
| for number, bucket in enumerate(buckets): | |
| for key, rows in sorted(bucket.items()): | |
| for path, logical, field, size, mtime in rows: | |
| hasher.update(json.dumps([number, key, str(path), logical, field, size, mtime]).encode() + b'\n') | |
| plan = dest / '.build' / name / 'PLAN.json' | |
| value = {'source_signature': hasher.hexdigest(), 'shards': len(buckets)} | |
| if plan.exists() and json.loads(plan.read_text()) != value: | |
| raise RuntimeError(f'Package inventory changed: {name}; choose a new output directory') | |
| plan.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = plan.with_suffix('.partial') | |
| temporary.write_text(json.dumps(value) + '\n') | |
| os.replace(temporary, plan) | |
| def write_index(dest, records): | |
| path = dest / 'index.sqlite.partial' | |
| if path.exists(): | |
| path.unlink() # Our own unfinished index only; rebuilt from completed journals. | |
| with sqlite3.connect(path) as conn: | |
| conn.execute('CREATE TABLE assets(path TEXT PRIMARY KEY, shard TEXT, offset INTEGER, size INTEGER, sample_key TEXT, field TEXT)') | |
| for record in records: | |
| rel = Path(record['file']) | |
| journal = dest / '.build' / rel.parent.name / (rel.stem + '.jsonl') | |
| with journal.open() as handle: | |
| conn.executemany('INSERT INTO assets VALUES (?,?,?,?,?,?)', (json.loads(line) for line in handle)) | |
| os.replace(path, dest / 'index.sqlite') | |
| def completed_records(dest): | |
| """Rebuild the global index from every committed package, including older ones.""" | |
| records = [] | |
| for marker in sorted((dest / '.build').glob('*/*.done.json')): | |
| record = json.loads(marker.read_text()) | |
| target = dest / record['file'] | |
| journal = marker.with_name(marker.name.replace('.done.json', '.jsonl')) | |
| if not target.is_file() or not journal.is_file(): | |
| raise FileNotFoundError(f'Incomplete committed package marker: {marker}') | |
| if digest(target) != record['sha256'] or digest(journal) != record['index_sha256']: | |
| raise RuntimeError(f'Committed package checksum mismatch: {marker}') | |
| records.append(record) | |
| return records | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument("--output", type=Path, required=True, help="Project-external export directory; can resume its completed shards") | |
| p.add_argument("--build", action="store_true", help="Write uncompressed WebDataset shards; otherwise print paths only") | |
| p.add_argument("--shards", type=int, default=128, help="Shard count for motion, TTS and mixed waveform packages") | |
| p.add_argument("--workers", type=int, default=4, help="CPU packing workers; no GPU usage") | |
| p.add_argument("--part", choices=("all", "data", "code"), default="all") | |
| p.add_argument("--motion-root", type=Path, default=ROOT.parent / "Psi0/dataset/seed_all") | |
| p.add_argument("--tts-root", type=Path, default=ROOT.parent.parent.parent / "buwendong/user/code/qwen3-tts") | |
| p.add_argument("--data-root", type=Path, default=ROOT.parent.parent / "videogen") | |
| p.add_argument("--third-party-root", type=Path, | |
| help="Parent containing production/ and training_scale_all_commands/") | |
| p.add_argument("--only-package", choices=(*PACKAGES, "seed_motion", "seed_tts", THIRD_PARTY_PACKAGE), | |
| help="Incrementally add/rebuild one package while preserving the global index") | |
| a = p.parse_args() | |
| if a.shards < 1 or a.workers < 1: | |
| p.error('shards/workers must be positive') | |
| out = a.output.expanduser().resolve() | |
| protected = [ROOT, a.motion_root.resolve(), a.tts_root.resolve(), a.data_root.resolve()] | |
| if a.third_party_root: | |
| protected.append(a.third_party_root.resolve()) | |
| if any(out == r or out in r.parents or r in out.parents for r in protected): | |
| raise ValueError("Output must be outside code and input data trees") | |
| jobs = dataset_jobs(a.motion_root, a.tts_root, a.data_root, a.third_party_root) | |
| if a.only_package: | |
| jobs = [(name, sources) for name, sources in jobs if name == a.only_package] | |
| if not jobs: | |
| raise ValueError(f'{a.only_package} needs its corresponding source argument') | |
| for name, sources in jobs if a.part != "code" else []: | |
| for path, _ in sources: | |
| if not path.exists(): | |
| raise FileNotFoundError(path) | |
| print(name, [(str(src), dst) for src, dst in sources], flush=True) | |
| if a.part != "data": | |
| entries = list(code_entries(ROOT)) | |
| print("code files/bytes (before gzip):", len(entries), sum(f.stat().st_size for f, _ in entries)) | |
| if not a.build: | |
| print("PLAN ONLY. Add --build to package; no HF upload is performed.") | |
| return | |
| if out.exists() and not (out / '.hfd-export.json').exists(): | |
| raise FileExistsError(f'Not an export owned by this tool: {out}') | |
| out.mkdir(parents=True, exist_ok=True) | |
| lock = (out / '.export.lock').open('a') | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| config = {'version': 2, 'shards': a.shards, | |
| # Existing exports predate optional incremental source roots. | |
| 'roots': [str(r) for r in protected[:4]]} | |
| ownership = out / '.hfd-export.json' | |
| if ownership.exists() and json.loads(ownership.read_text()) != config: | |
| raise ValueError('Export configuration changed; choose a new directory') | |
| ownership.write_text(json.dumps(config) + '\n') | |
| if a.part != "code": | |
| dest = out / "hfd_data" | |
| records = [] | |
| for name, sources in jobs: | |
| count = a.shards if name in {'seed_motion', 'seed_tts', PACKAGES[0], THIRD_PARTY_PACKAGE} else (min(a.shards, 32) if name.endswith('hesitation_pause_v1') else 1) | |
| print(f'[scan] {name}: preparing {count} shards', flush=True) | |
| buckets, raw = plan_shards(name, sources, count) | |
| verify_package_plan(dest, name, buckets) | |
| print(f'[plan] {name}: {raw / 2**30:.3f} GiB logical bytes', flush=True) | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool: | |
| futures = [pool.submit(pack_shard, dest, name, i, bucket) for i, bucket in enumerate(buckets) if bucket] | |
| records.extend(f.result() for f in futures) | |
| del buckets | |
| # An incremental invocation must not erase the older packages from the | |
| # shared SQLite index/manifest/checksum file. | |
| records = completed_records(dest) | |
| write_index(dest, records) | |
| shutil.copyfile(ROOT / "doc/hf_audio_motion/DATA_README.md", dest / "README.md") | |
| shutil.copyfile(ROOT / "doc/hf_audio_motion/PATH_MAP.json", dest / "PATH_MAP.json") | |
| shutil.copyfile(ROOT / 'scripts/data/read_hf_audio_motion.py', dest / 'read_hf_audio_motion.py') | |
| (dest / "normalization").mkdir(exist_ok=True) | |
| shutil.copyfile(ROOT / "assets/stats/seed_all_g1_motion39_textop_stats.json", | |
| dest / "normalization/seed_all_g1_motion39_textop_stats.json") | |
| (dest / "MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n") | |
| checks = ''.join(f"{r['sha256']} {r['file']}\n" for r in records) | |
| for item in ['index.sqlite', 'PATH_MAP.json', 'read_hf_audio_motion.py', 'normalization/seed_all_g1_motion39_textop_stats.json', 'MANIFEST.json']: | |
| checks += f'{digest(dest / item)} {item}\n' | |
| (dest / "SHA256SUMS").write_text(checks) | |
| if a.part != "data": | |
| dest = out / "hfd" | |
| if (dest / 'code/hfd_source.tar.gz').exists(): | |
| snapshot = json.loads((dest / 'SNAPSHOT.json').read_text()) | |
| if digest(dest / 'code/hfd_source.tar.gz') != snapshot['sha256']: | |
| raise RuntimeError('Previous code snapshot checksum mismatch') | |
| print(f'Preserved previously completed code snapshot: {dest}. Use a new output for newer code.') | |
| return | |
| # Keep the upstream notice in the exported copy even if locally deleted; | |
| # do not restore or alter the user's working-tree LICENSE. | |
| license_bytes = (ROOT / "LICENSE").read_bytes() if (ROOT / "LICENSE").exists() else subprocess.check_output( | |
| ["git", "show", "HEAD:LICENSE"], cwd=ROOT) | |
| snapshot = {"git_head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT).decode().strip(), | |
| "created_unix": time.time(), "source": "working tree, including uncommitted source changes", | |
| "sanitization": "Historical commented access tokens redacted in exported copy; executable token matches rejected", | |
| "excluded": ["environment", "credentials", "runs", "datasets", "models", "git history", "DIALOG", "historical experiment outputs"]} | |
| record = write_archive(dest / "code/hfd_source.tar.gz", code_entries(ROOT), | |
| {"HFD/LICENSE": license_bytes, "HFD/SNAPSHOT.json": json.dumps(snapshot, indent=2).encode()}, resume_partial=True) | |
| shutil.copyfile(ROOT / "doc/hf_audio_motion/CODE_README.md", dest / "README.md") | |
| (dest / "SNAPSHOT.json").write_text(json.dumps({**snapshot, **record}, indent=2) + "\n") | |
| (dest / "SHA256SUMS").write_text(f"{record['sha256']} code/hfd_source.tar.gz\n") | |
| print(f"Complete: {out}. Review MANIFEST/SNAPSHOT and SHA256SUMS before uploading.") | |
| if __name__ == "__main__": | |
| main() | |