Download scripts/data/package_hf_smplx.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/package_hf_smplx.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/package_hf_smplx.py
-
curl -L -o package_hf_smplx.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/package_hf_smplx.py
11.6 kB
| #!/usr/bin/env python3 | |
| """Add validated SMPL-X sources to the EXISTING HFD tar/index namespace. | |
| Builds only new shards in an external staging directory; never repacks audio. | |
| Upload is a single parent-revision-guarded commit to a private repository. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from concurrent.futures import ThreadPoolExecutor | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shutil | |
| import sqlite3 | |
| import time | |
| import pandas as pd | |
| from huggingface_hub import HfApi, hf_hub_download | |
| from package_hf_audio_motion import digest, pack_shard, plan_shards, verify_package_plan, write_index | |
| REPO = 'Tinghong-Ye22/hfd_data' | |
| LOGICAL = 'motion/seed_all/smplx_amass' | |
| ROOT = Path(__file__).resolve().parents[2] | |
| def save(path, value): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_suffix(path.suffix + '.partial') | |
| tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') | |
| os.replace(tmp, path) | |
| def pair_sources(source, base, metadata, tts_path): | |
| """Join by identity, NEVER by CSV/parquet row order or caption alone.""" | |
| table = pd.read_parquet(source / 'manifest.parquet') | |
| seed = pd.read_csv(metadata, keep_default_na=False) | |
| if table.filename.duplicated().any() or seed.filename.duplicated().any(): | |
| raise ValueError('Non-unique motion identities') | |
| if set(table.filename) != set(seed.filename): | |
| raise ValueError('SMPL-X / G1 metadata identities differ') | |
| tts = {} | |
| for line in tts_path.read_text().splitlines(): | |
| row = json.loads(line) | |
| name = row['filename'] | |
| if name in tts: | |
| raise ValueError(f'Duplicate TTS identity: {name}') | |
| tts[name] = row | |
| with sqlite3.connect(f'file:{base / "index.sqlite"}?mode=ro', uri=True) as conn: | |
| assets = {r[0] for r in conn.execute('SELECT path FROM assets')} | |
| descriptions = seed.set_index('filename') | |
| rows = [] | |
| for r in table.to_dict('records'): | |
| name = r['filename'] | |
| original = descriptions.loc[name] | |
| if original.move_g1_path != r['move_g1_path']: | |
| raise ValueError(f'G1 path mismatch: {name}') | |
| relative = r['move_g1_path'].replace('g1/csv/', 'g1/csv_sonic/').removesuffix('.csv') | |
| folder = 'motion/seed_all/' + relative | |
| for field in ('joint_pos.csv', 'body_pos.csv', 'body_quat.csv'): | |
| if folder + '/' + field not in assets: | |
| raise ValueError(f'G1 not in existing HF index: {folder}/{field}') | |
| voice = tts.get(name) | |
| audio_dir = ('audio/seed_tts/seed_audio/' + Path(voice['audio_path']).name) if voice else None | |
| # Build the audio folder map once below (avoid 142k full-index scans). | |
| rows.append(dict(motion_id=name, g1_folder=folder, | |
| smplx_path=LOGICAL + '/' + r['output_path'], | |
| smplx_sha256=r['output_sha256'], | |
| source_bvh_sha256=r['source_bvh_sha256'], | |
| smplx_frames=int(r['output_frames']), smplx_fps=float(r['fps']), | |
| source_frames=int(r['source_frames']), | |
| source_frame_time=float(r['source_frame_time']), | |
| instruction_en=voice.get('content_short_description') if voice else None, | |
| instruction_zh=voice.get('content_short_description_zh') if voice else None, | |
| audio_dir=audio_dir)) | |
| audio_folders = {} | |
| for path in assets: | |
| if path.startswith('audio/seed_tts/seed_audio/') and path.endswith('.wav'): | |
| folder, voice = path.rsplit('/', 1) | |
| audio_folders.setdefault(folder, []).append(voice.removesuffix('.wav')) | |
| for row in rows: | |
| row['voices'] = sorted(audio_folders.get(row['audio_dir'], [])) | |
| row['has_instruction_audio'] = bool(row['voices']) | |
| return pd.DataFrame(rows) | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--source', required=True, type=Path) | |
| p.add_argument('--metadata', required=True, type=Path) | |
| p.add_argument('--tts-metadata', required=True, type=Path) | |
| p.add_argument('--output', required=True, type=Path) | |
| p.add_argument('--workers', type=int, default=16) | |
| p.add_argument('--upload', action='store_true') | |
| a = p.parse_args() | |
| api = HfApi() | |
| info = api.repo_info(REPO, repo_type='dataset') | |
| if not info.private: | |
| raise RuntimeError('Refusing gated SMPL-X delivery to a public repository') | |
| receipt = json.loads((a.source / 'validation_summary.json').read_text()) | |
| if receipt['status'] != 'validated' or receipt['success'] != receipt['expected'] or receipt['failures']: | |
| raise RuntimeError('Conversion is not completely validated') | |
| out = a.output.resolve() | |
| if out == a.source.resolve() or a.source.resolve() in out.parents or ROOT == out or ROOT in out.parents: | |
| raise ValueError('Use a separate project-external staging directory') | |
| if out.exists() and not (out / '.build/human_export.json').exists(): | |
| raise FileExistsError(out) | |
| state_path = out / '.build/human_export.json' | |
| if state_path.exists(): | |
| state = json.loads(state_path.read_text()) | |
| if state['parent_revision'] != info.sha: | |
| raise RuntimeError('HF changed since export began; review before rebuilding index') | |
| else: | |
| state = dict(parent_revision=info.sha, source=str(a.source.resolve()), stage='prepare') | |
| save(state_path, state) | |
| base = out / '.build/base' | |
| base.mkdir(parents=True, exist_ok=True) | |
| for name in ('index.sqlite', 'MANIFEST.json', 'PATH_MAP.json', 'SHA256SUMS', 'README.md'): | |
| src = hf_hub_download(REPO, name, repo_type='dataset', revision=state['parent_revision']) | |
| shutil.copyfile(src, base / name) | |
| paired = pair_sources(a.source, base, a.metadata, a.tts_metadata) | |
| if len(paired) != receipt['expected']: | |
| raise ValueError('Pair coverage differs from validated conversion') | |
| meta = out / 'metadata' | |
| meta.mkdir(exist_ok=True) | |
| paired.to_parquet(meta / 'seed_g1_smplx_pairs.parquet', index=False) | |
| mapping_summary = dict(motions=len(paired), with_audio=int(paired.has_instruction_audio.sum()), | |
| eight_voice_motions=int(paired.voices.map(len).eq(8).sum()), | |
| audio_directories=int(paired.audio_dir.nunique()), | |
| conversion=receipt, parent_revision=state['parent_revision']) | |
| save(meta / 'seed_smplx_summary.json', mapping_summary) | |
| sources = [(a.source / 'motions', LOGICAL + '/motions')] | |
| for name in ('manifest.parquet', 'validation_summary.json', 'README.md', | |
| 'conversion_receipt.json', 'identity_receipt.json', 'source_archive_receipt.json'): | |
| sources.append((a.source / name, LOGICAL + '/' + name)) | |
| buckets, raw_bytes = plan_shards('seed_smplx', sources, 128) | |
| verify_package_plan(out, 'seed_smplx', buckets) | |
| state.update(stage='packing', raw_bytes=raw_bytes, shards_done=0) | |
| save(state_path, state) | |
| with ThreadPoolExecutor(max_workers=a.workers) as pool: | |
| futures = [pool.submit(pack_shard, out, 'seed_smplx', i, b) for i, b in enumerate(buckets)] | |
| records = [] | |
| from concurrent.futures import as_completed | |
| for f in as_completed(futures): | |
| records.append(f.result()) | |
| state['shards_done'] = len(records) | |
| save(state_path, state) | |
| records.sort(key=lambda r: r['file']) | |
| write_index(out, records) | |
| # The shared root index retains EVERY old asset, while adding the new namespace. | |
| with sqlite3.connect(out / 'index.sqlite') as conn: | |
| conn.execute('ATTACH DATABASE ? AS old', (str(base / 'index.sqlite'),)) | |
| conn.execute('INSERT INTO assets SELECT * FROM old.assets') | |
| conn.commit() | |
| count = conn.execute('SELECT count(*) FROM assets').fetchone()[0] | |
| paths = json.loads((base / 'PATH_MAP.json').read_text()) | |
| paths['prefixes'][str(a.source.resolve())] = LOGICAL | |
| save(out / 'PATH_MAP.json', paths) | |
| manifest = json.loads((base / 'MANIFEST.json').read_text()) | |
| if not isinstance(manifest, list): | |
| raise ValueError('Unexpected existing MANIFEST schema; do not overwrite it') | |
| if any(r['file'].startswith('shards/seed_smplx/') for r in manifest): | |
| raise ValueError('SMPL-X package already exists; explicit version review required') | |
| manifest.extend(records) | |
| save(out / 'MANIFEST.json', manifest) | |
| shutil.copyfile(ROOT / 'scripts/data/read_hf_audio_motion.py', out / 'read_hf_audio_motion.py') | |
| shutil.copyfile(ROOT / 'doc/hf_audio_motion/SMPLX_HANDOFF.md', out / 'SMPLX_HANDOFF.md') | |
| readme = (base / 'README.md').read_text() | |
| readme += '\n\n## Human motion 增量包(2026-10-13)\n\n' + ( | |
| '新增 `shards/seed_smplx/`(128 个 tar),与原 G1/音频共用根目录索引;无需重复下载语音。' | |
| '142,220 条 AMASS SMPL-X 原始动作,约 19 GiB。对应表:' | |
| '`metadata/seed_g1_smplx_pairs.parquet`。详细格式、限制和 session 接入见 ' | |
| '[SMPLX_HANDOFF.md](SMPLX_HANDOFF.md)。**源动作交付不等于已验证的完整人体 session 数据集。**\n') | |
| (out / 'README.md').write_text(readme) | |
| # Exclude overwritten metadata checksums, retain untouched old assets verbatim. | |
| replaced = {'index.sqlite','PATH_MAP.json','MANIFEST.json','README.md','read_hf_audio_motion.py'} | |
| old_lines = [line for line in (base / 'SHA256SUMS').read_text().splitlines() | |
| if line.strip() and line.split(maxsplit=1)[1].lstrip('*').removeprefix('./') not in replaced] | |
| new_files = sorted(f for f in out.rglob('*') if f.is_file() and '.build' not in f.relative_to(out).parts | |
| and f.name != 'SHA256SUMS') | |
| checks = [f'{digest(f)} {f.relative_to(out).as_posix()}' for f in new_files] | |
| (out / 'SHA256SUMS').write_text('\n'.join(old_lines + checks) + '\n') | |
| # Read each new payload THROUGH the merged index and verify its conversion hash. | |
| from read_hf_audio_motion import ShardStore | |
| store = ShardStore(out) | |
| try: | |
| for row in paired.itertuples(): | |
| if hashlib.sha256(store.read(row.smplx_path)).hexdigest() != row.smplx_sha256: | |
| raise ValueError(f'Indexed payload checksum failed: {row.motion_id}') | |
| finally: | |
| store.close() | |
| state.update(stage='verified', assets=count, motions=len(paired), updated_at=time.time()) | |
| save(state_path, state) | |
| print(json.dumps(state), flush=True) | |
| if a.upload: | |
| if not api.repo_info(REPO, repo_type='dataset').private: | |
| raise RuntimeError('Repository is no longer private') | |
| state['stage'] = 'uploading' | |
| save(state_path, state) | |
| try: | |
| result = api.upload_folder(repo_id=REPO, repo_type='dataset', folder_path=out, | |
| ignore_patterns=['.build/**'], parent_commit=state['parent_revision'], | |
| commit_message='Add validated paired BONES-SEED SMPL-X source shards') | |
| except Exception as exc: | |
| state.update(stage='upload_failed', error_type=type(exc).__name__, | |
| private_storage_limit=('Private storage limit reached' in str(exc)), | |
| updated_at=time.time()) | |
| save(state_path, state) | |
| raise | |
| state.update(stage='uploaded', commit=result.oid, url=result.commit_url, updated_at=time.time()) | |
| save(state_path, state) | |
| print(json.dumps(state), flush=True) | |
| if __name__ == '__main__': | |
| main() | |