code-HFD / scripts /data /package_hf_audio_motion.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
20 kB
#!/usr/bin/env python3
"""Offline HF handoff packaging. Default is a read-only plan; never uploads."""
from __future__ import annotations
import argparse
import concurrent.futures
import fcntl
import hashlib
import io
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
import tarfile
import time
ROOT = Path(__file__).resolve().parents[2]
PACKAGES = (
"hfd_audio_command_continuous_office_v1",
"hfd_control_commands_v1",
"hfd_backchannel_commands_v2_short",
"hfd_hesitation_pause_v1",
"hfd_hesitation_pause_v2",
)
THIRD_PARTY_PACKAGE = "hfd_third_party_multi_v5"
SKIP_DIRS = {".git", ".venv-psi", "__pycache__", ".pytest_cache", ".cache",
".progress", "logs", "gt_review", "runs", "checkpoints"}
CODE_DIRS = ("src", "scripts", "tests", "assets", "backup")
CODE_FILES = ("README.md", "pyproject.toml", "uv.lock", "posttrain-full-duplex-audio-command.sh",
"infer_render_auto.sh")
def files_under(path):
"""Never follow a link into another dataset, environment or private tree."""
path = Path(path)
if path.is_symlink():
raise ValueError(f"Refusing symlink: {path}")
if path.is_file():
yield path
return
if not path.is_dir():
raise FileNotFoundError(path)
for directory, dirs, names in os.walk(path, followlinks=False):
dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)
for name in [*dirs, *names]:
item = Path(directory) / name
if item.is_symlink():
raise ValueError(f"Refusing symlink: {item}")
for name in sorted(names):
if (name.startswith(".env") or name in {"id_rsa", "id_ed25519"}
or name.endswith((".pyc", ".pyo", ".key", ".pem"))):
continue
yield Path(directory) / name
def code_entries(root):
roots = [root / p for p in (*CODE_DIRS, *CODE_FILES)]
roots.append(root / "doc/hf_audio_motion")
for base in roots:
for path in files_under(base):
if path.suffix.lower() in {'.pt', '.pth', '.safetensors', '.wav', '.mp4', '.pkl', '.sqlite', '.bin'}:
continue
yield path, "HFD/" + path.relative_to(root).as_posix()
def digest(path):
h = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(8 * 1024 * 1024), b""):
h.update(block)
return h.hexdigest()
def safe_code_bytes(path):
"""Remove historical commented tokens in export only; reject executable secrets."""
value = path.read_bytes()
if re.search(rb'-----BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY-----', value):
raise ValueError(f'Private key detected; not exporting {path}')
token = re.compile(rb'(?<![A-Za-z0-9_])(?:hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9_-]{20,})')
lines = []
for line in value.splitlines(keepends=True):
if token.search(line):
if not line.lstrip().startswith(b'#'):
raise ValueError(f'Possible executable credential; review {path}')
line = token.sub(b'[REDACTED_TOKEN]', line)
lines.append(line)
return b''.join(lines)
def write_archive(target, entries, extras=None, resume_partial=False):
"""Atomic archive; no destructive replacement of an earlier export."""
if target.exists():
raise FileExistsError(target)
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(target.name + ".partial")
if temporary.exists() and not resume_partial:
raise FileExistsError(temporary)
count = raw_bytes = 0
with tarfile.open(temporary, "w:gz", compresslevel=1, dereference=False) as archive:
for path, name in entries:
if path.is_symlink() or not path.is_file():
raise ValueError(f"Not a regular file: {path}")
before = path.stat()
value = safe_code_bytes(path)
info = archive.gettarinfo(str(path), arcname=name)
info.size = len(value)
archive.addfile(info, io.BytesIO(value))
after = path.stat()
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
raise RuntimeError(f"Source changed during packaging: {path}")
count += 1
raw_bytes += len(value)
if count % 20000 == 0:
print(f"[pack] {target.name}: {count:,} files", flush=True)
for name, value in (extras or {}).items():
info = tarfile.TarInfo(name)
info.size = len(value)
info.mode = 0o644
archive.addfile(info, io.BytesIO(value))
count += 1
raw_bytes += len(value)
os.replace(temporary, target)
result = {"file": target.name, "files": count, "raw_bytes": raw_bytes,
"archive_bytes": target.stat().st_size, "sha256": digest(target)}
print(json.dumps(result), flush=True)
return result
def dataset_jobs(motion, tts, data, third_party=None):
# Preserve source bytes/IDs. Absolute paths are handled by the receiving loader.
jobs = [("seed_motion", [(motion / "g1/csv_sonic", "motion/seed_all/g1/csv_sonic"),
(motion / "metadata/seed_metadata_v004.csv", "motion/seed_all/metadata/seed_metadata_v004.csv")]),
("seed_tts", [(tts / "seed_audio", "audio/seed_tts/seed_audio"),
(tts / "metadata.jsonl", "audio/seed_tts/metadata.jsonl")])]
jobs += [(name, [(data / name, "audio/" + name)]) for name in PACKAGES]
if third_party is not None:
jobs.append((THIRD_PARTY_PACKAGE, [
(third_party / "production", "audio/hfd_third_party_multi_v5/production"),
(third_party / "training_scale_all_commands",
"audio/hfd_third_party_multi_v5/training_scale_all_commands"),
]))
return jobs
def mapped_entries(sources):
for source, destination in sources:
for path in files_under(source):
name = destination if source.is_file() else destination + "/" + path.relative_to(source).as_posix()
yield path, name
def plan_shards(name, sources, count):
"""One motion folder per sample; other files are independent samples.
Stable hash assignment avoids depending on filesystem traversal order.
Metadata JSONL/CSV remains byte-identical and is addressable via SQLite.
"""
buckets = [dict() for _ in range(count)]
total = 0
for path, logical in mapped_entries(sources):
sample_path = logical.rsplit('/', 1)[0] if name == 'seed_motion' and '/g1/csv_sonic/' in logical else logical
key = hashlib.sha256(sample_path.encode()).hexdigest()[:40]
bucket = buckets[int(key[:16], 16) % count]
field = path.name.lower() if sample_path != logical else 'payload' + (path.suffix.lower() or '.bin')
if any(ch not in 'abcdefghijklmnopqrstuvwxyz0123456789_.' for ch in field):
field = 'payload.bin'
st = path.stat()
bucket.setdefault(key, []).append((path, logical, field, st.st_size, st.st_mtime_ns))
total += st.st_size
return buckets, total
def pack_shard(dest, name, number, samples):
"""Uncompressed, contiguous WDS samples + offsets. Resume verified shards."""
rel = f'shards/{name}/{number:05d}.tar'
target = dest / rel
journal = dest / '.build' / name / f'{number:05d}.jsonl'
marker = journal.with_suffix('.done.json')
inventory = [(k, str(p), n, f, size, mt) for k, rows in sorted(samples.items()) for p, n, f, size, mt in rows]
signature = hashlib.sha256(json.dumps(inventory, ensure_ascii=False).encode()).hexdigest()
if marker.exists():
record = json.loads(marker.read_text())
if record['source_signature'] != signature:
raise RuntimeError(f'Sources changed: {rel}; choose a NEW export directory')
if not target.exists() or not journal.exists() or digest(target) != record['sha256'] or digest(journal) != record['index_sha256']:
raise RuntimeError(f'Completed shard corrupted: {rel}')
print(f'[resume] {rel}', flush=True)
return record
target.parent.mkdir(parents=True, exist_ok=True)
journal.parent.mkdir(parents=True, exist_ok=True)
partial = target.with_suffix('.tar.partial')
count = raw = 0
with tarfile.open(partial, 'w', format=tarfile.USTAR_FORMAT) as tar, journal.open('w') as index:
for key, rows in sorted(samples.items()):
fields = [row[2] for row in rows]
if len(fields) != len(set(fields)) or 'json' in fields:
raise ValueError(f'Duplicate WDS fields: {key}')
meta = {'key': key, 'package': name, 'paths': {row[2]: row[1] for row in rows}}
content = json.dumps(meta, ensure_ascii=False).encode()
info = tarfile.TarInfo(key + '.json')
info.size = len(content)
tar.addfile(info, io.BytesIO(content))
for path, logical, field, size, mtime in rows:
before = path.stat()
if path.is_symlink() or (before.st_size, before.st_mtime_ns) != (size, mtime):
raise RuntimeError(f'Source changed since planning: {path}')
info = tarfile.TarInfo(key + '.' + field)
info.size = size
offset = tar.offset + 512 # USTAR, short ASCII names: exactly one header block.
with path.open('rb') as handle:
tar.addfile(info, handle)
after = path.stat()
if (after.st_size, after.st_mtime_ns) != (size, mtime):
raise RuntimeError(f'Source changed during packaging: {path}')
index.write(json.dumps([logical, rel, offset, size, key, field], ensure_ascii=False) + '\n')
count += 1
raw += size
os.replace(partial, target)
record = {'file': rel, 'files': count, 'samples': len(samples), 'raw_bytes': raw,
'archive_bytes': target.stat().st_size, 'sha256': digest(target),
'source_signature': signature, 'index_sha256': digest(journal)}
tmp = marker.with_suffix('.partial')
tmp.write_text(json.dumps(record) + '\n')
os.replace(tmp, marker) # Only this marker constitutes a completed shard.
print(f'[done] {rel}: {len(samples):,} samples, {raw / 2**20:.1f} MiB', flush=True)
return record
def verify_package_plan(dest, name, buckets):
"""Also detect removals that would make a previously completed shard empty."""
hasher = hashlib.sha256()
for number, bucket in enumerate(buckets):
for key, rows in sorted(bucket.items()):
for path, logical, field, size, mtime in rows:
hasher.update(json.dumps([number, key, str(path), logical, field, size, mtime]).encode() + b'\n')
plan = dest / '.build' / name / 'PLAN.json'
value = {'source_signature': hasher.hexdigest(), 'shards': len(buckets)}
if plan.exists() and json.loads(plan.read_text()) != value:
raise RuntimeError(f'Package inventory changed: {name}; choose a new output directory')
plan.parent.mkdir(parents=True, exist_ok=True)
temporary = plan.with_suffix('.partial')
temporary.write_text(json.dumps(value) + '\n')
os.replace(temporary, plan)
def write_index(dest, records):
path = dest / 'index.sqlite.partial'
if path.exists():
path.unlink() # Our own unfinished index only; rebuilt from completed journals.
with sqlite3.connect(path) as conn:
conn.execute('CREATE TABLE assets(path TEXT PRIMARY KEY, shard TEXT, offset INTEGER, size INTEGER, sample_key TEXT, field TEXT)')
for record in records:
rel = Path(record['file'])
journal = dest / '.build' / rel.parent.name / (rel.stem + '.jsonl')
with journal.open() as handle:
conn.executemany('INSERT INTO assets VALUES (?,?,?,?,?,?)', (json.loads(line) for line in handle))
os.replace(path, dest / 'index.sqlite')
def completed_records(dest):
"""Rebuild the global index from every committed package, including older ones."""
records = []
for marker in sorted((dest / '.build').glob('*/*.done.json')):
record = json.loads(marker.read_text())
target = dest / record['file']
journal = marker.with_name(marker.name.replace('.done.json', '.jsonl'))
if not target.is_file() or not journal.is_file():
raise FileNotFoundError(f'Incomplete committed package marker: {marker}')
if digest(target) != record['sha256'] or digest(journal) != record['index_sha256']:
raise RuntimeError(f'Committed package checksum mismatch: {marker}')
records.append(record)
return records
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--output", type=Path, required=True, help="Project-external export directory; can resume its completed shards")
p.add_argument("--build", action="store_true", help="Write uncompressed WebDataset shards; otherwise print paths only")
p.add_argument("--shards", type=int, default=128, help="Shard count for motion, TTS and mixed waveform packages")
p.add_argument("--workers", type=int, default=4, help="CPU packing workers; no GPU usage")
p.add_argument("--part", choices=("all", "data", "code"), default="all")
p.add_argument("--motion-root", type=Path, default=ROOT.parent / "Psi0/dataset/seed_all")
p.add_argument("--tts-root", type=Path, default=ROOT.parent.parent.parent / "buwendong/user/code/qwen3-tts")
p.add_argument("--data-root", type=Path, default=ROOT.parent.parent / "videogen")
p.add_argument("--third-party-root", type=Path,
help="Parent containing production/ and training_scale_all_commands/")
p.add_argument("--only-package", choices=(*PACKAGES, "seed_motion", "seed_tts", THIRD_PARTY_PACKAGE),
help="Incrementally add/rebuild one package while preserving the global index")
a = p.parse_args()
if a.shards < 1 or a.workers < 1:
p.error('shards/workers must be positive')
out = a.output.expanduser().resolve()
protected = [ROOT, a.motion_root.resolve(), a.tts_root.resolve(), a.data_root.resolve()]
if a.third_party_root:
protected.append(a.third_party_root.resolve())
if any(out == r or out in r.parents or r in out.parents for r in protected):
raise ValueError("Output must be outside code and input data trees")
jobs = dataset_jobs(a.motion_root, a.tts_root, a.data_root, a.third_party_root)
if a.only_package:
jobs = [(name, sources) for name, sources in jobs if name == a.only_package]
if not jobs:
raise ValueError(f'{a.only_package} needs its corresponding source argument')
for name, sources in jobs if a.part != "code" else []:
for path, _ in sources:
if not path.exists():
raise FileNotFoundError(path)
print(name, [(str(src), dst) for src, dst in sources], flush=True)
if a.part != "data":
entries = list(code_entries(ROOT))
print("code files/bytes (before gzip):", len(entries), sum(f.stat().st_size for f, _ in entries))
if not a.build:
print("PLAN ONLY. Add --build to package; no HF upload is performed.")
return
if out.exists() and not (out / '.hfd-export.json').exists():
raise FileExistsError(f'Not an export owned by this tool: {out}')
out.mkdir(parents=True, exist_ok=True)
lock = (out / '.export.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
config = {'version': 2, 'shards': a.shards,
# Existing exports predate optional incremental source roots.
'roots': [str(r) for r in protected[:4]]}
ownership = out / '.hfd-export.json'
if ownership.exists() and json.loads(ownership.read_text()) != config:
raise ValueError('Export configuration changed; choose a new directory')
ownership.write_text(json.dumps(config) + '\n')
if a.part != "code":
dest = out / "hfd_data"
records = []
for name, sources in jobs:
count = a.shards if name in {'seed_motion', 'seed_tts', PACKAGES[0], THIRD_PARTY_PACKAGE} else (min(a.shards, 32) if name.endswith('hesitation_pause_v1') else 1)
print(f'[scan] {name}: preparing {count} shards', flush=True)
buckets, raw = plan_shards(name, sources, count)
verify_package_plan(dest, name, buckets)
print(f'[plan] {name}: {raw / 2**30:.3f} GiB logical bytes', flush=True)
with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:
futures = [pool.submit(pack_shard, dest, name, i, bucket) for i, bucket in enumerate(buckets) if bucket]
records.extend(f.result() for f in futures)
del buckets
# An incremental invocation must not erase the older packages from the
# shared SQLite index/manifest/checksum file.
records = completed_records(dest)
write_index(dest, records)
shutil.copyfile(ROOT / "doc/hf_audio_motion/DATA_README.md", dest / "README.md")
shutil.copyfile(ROOT / "doc/hf_audio_motion/PATH_MAP.json", dest / "PATH_MAP.json")
shutil.copyfile(ROOT / 'scripts/data/read_hf_audio_motion.py', dest / 'read_hf_audio_motion.py')
(dest / "normalization").mkdir(exist_ok=True)
shutil.copyfile(ROOT / "assets/stats/seed_all_g1_motion39_textop_stats.json",
dest / "normalization/seed_all_g1_motion39_textop_stats.json")
(dest / "MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n")
checks = ''.join(f"{r['sha256']} {r['file']}\n" for r in records)
for item in ['index.sqlite', 'PATH_MAP.json', 'read_hf_audio_motion.py', 'normalization/seed_all_g1_motion39_textop_stats.json', 'MANIFEST.json']:
checks += f'{digest(dest / item)} {item}\n'
(dest / "SHA256SUMS").write_text(checks)
if a.part != "data":
dest = out / "hfd"
if (dest / 'code/hfd_source.tar.gz').exists():
snapshot = json.loads((dest / 'SNAPSHOT.json').read_text())
if digest(dest / 'code/hfd_source.tar.gz') != snapshot['sha256']:
raise RuntimeError('Previous code snapshot checksum mismatch')
print(f'Preserved previously completed code snapshot: {dest}. Use a new output for newer code.')
return
# Keep the upstream notice in the exported copy even if locally deleted;
# do not restore or alter the user's working-tree LICENSE.
license_bytes = (ROOT / "LICENSE").read_bytes() if (ROOT / "LICENSE").exists() else subprocess.check_output(
["git", "show", "HEAD:LICENSE"], cwd=ROOT)
snapshot = {"git_head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT).decode().strip(),
"created_unix": time.time(), "source": "working tree, including uncommitted source changes",
"sanitization": "Historical commented access tokens redacted in exported copy; executable token matches rejected",
"excluded": ["environment", "credentials", "runs", "datasets", "models", "git history", "DIALOG", "historical experiment outputs"]}
record = write_archive(dest / "code/hfd_source.tar.gz", code_entries(ROOT),
{"HFD/LICENSE": license_bytes, "HFD/SNAPSHOT.json": json.dumps(snapshot, indent=2).encode()}, resume_partial=True)
shutil.copyfile(ROOT / "doc/hf_audio_motion/CODE_README.md", dest / "README.md")
(dest / "SNAPSHOT.json").write_text(json.dumps({**snapshot, **record}, indent=2) + "\n")
(dest / "SHA256SUMS").write_text(f"{record['sha256']} code/hfd_source.tar.gz\n")
print(f"Complete: {out}. Review MANIFEST/SNAPSHOT and SHA256SUMS before uploading.")
if __name__ == "__main__":
main()