File size: 20,015 Bytes
9fc9873 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 | #!/usr/bin/env python3
"""Offline HF handoff packaging. Default is a read-only plan; never uploads."""
from __future__ import annotations
import argparse
import concurrent.futures
import fcntl
import hashlib
import io
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
import tarfile
import time
ROOT = Path(__file__).resolve().parents[2]
PACKAGES = (
"hfd_audio_command_continuous_office_v1",
"hfd_control_commands_v1",
"hfd_backchannel_commands_v2_short",
"hfd_hesitation_pause_v1",
"hfd_hesitation_pause_v2",
)
THIRD_PARTY_PACKAGE = "hfd_third_party_multi_v5"
SKIP_DIRS = {".git", ".venv-psi", "__pycache__", ".pytest_cache", ".cache",
".progress", "logs", "gt_review", "runs", "checkpoints"}
CODE_DIRS = ("src", "scripts", "tests", "assets", "backup")
CODE_FILES = ("README.md", "pyproject.toml", "uv.lock", "posttrain-full-duplex-audio-command.sh",
"infer_render_auto.sh")
def files_under(path):
"""Never follow a link into another dataset, environment or private tree."""
path = Path(path)
if path.is_symlink():
raise ValueError(f"Refusing symlink: {path}")
if path.is_file():
yield path
return
if not path.is_dir():
raise FileNotFoundError(path)
for directory, dirs, names in os.walk(path, followlinks=False):
dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)
for name in [*dirs, *names]:
item = Path(directory) / name
if item.is_symlink():
raise ValueError(f"Refusing symlink: {item}")
for name in sorted(names):
if (name.startswith(".env") or name in {"id_rsa", "id_ed25519"}
or name.endswith((".pyc", ".pyo", ".key", ".pem"))):
continue
yield Path(directory) / name
def code_entries(root):
roots = [root / p for p in (*CODE_DIRS, *CODE_FILES)]
roots.append(root / "doc/hf_audio_motion")
for base in roots:
for path in files_under(base):
if path.suffix.lower() in {'.pt', '.pth', '.safetensors', '.wav', '.mp4', '.pkl', '.sqlite', '.bin'}:
continue
yield path, "HFD/" + path.relative_to(root).as_posix()
def digest(path):
h = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(8 * 1024 * 1024), b""):
h.update(block)
return h.hexdigest()
def safe_code_bytes(path):
"""Remove historical commented tokens in export only; reject executable secrets."""
value = path.read_bytes()
if re.search(rb'-----BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY-----', value):
raise ValueError(f'Private key detected; not exporting {path}')
token = re.compile(rb'(?<![A-Za-z0-9_])(?:hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9_-]{20,})')
lines = []
for line in value.splitlines(keepends=True):
if token.search(line):
if not line.lstrip().startswith(b'#'):
raise ValueError(f'Possible executable credential; review {path}')
line = token.sub(b'[REDACTED_TOKEN]', line)
lines.append(line)
return b''.join(lines)
def write_archive(target, entries, extras=None, resume_partial=False):
"""Atomic archive; no destructive replacement of an earlier export."""
if target.exists():
raise FileExistsError(target)
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(target.name + ".partial")
if temporary.exists() and not resume_partial:
raise FileExistsError(temporary)
count = raw_bytes = 0
with tarfile.open(temporary, "w:gz", compresslevel=1, dereference=False) as archive:
for path, name in entries:
if path.is_symlink() or not path.is_file():
raise ValueError(f"Not a regular file: {path}")
before = path.stat()
value = safe_code_bytes(path)
info = archive.gettarinfo(str(path), arcname=name)
info.size = len(value)
archive.addfile(info, io.BytesIO(value))
after = path.stat()
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
raise RuntimeError(f"Source changed during packaging: {path}")
count += 1
raw_bytes += len(value)
if count % 20000 == 0:
print(f"[pack] {target.name}: {count:,} files", flush=True)
for name, value in (extras or {}).items():
info = tarfile.TarInfo(name)
info.size = len(value)
info.mode = 0o644
archive.addfile(info, io.BytesIO(value))
count += 1
raw_bytes += len(value)
os.replace(temporary, target)
result = {"file": target.name, "files": count, "raw_bytes": raw_bytes,
"archive_bytes": target.stat().st_size, "sha256": digest(target)}
print(json.dumps(result), flush=True)
return result
def dataset_jobs(motion, tts, data, third_party=None):
# Preserve source bytes/IDs. Absolute paths are handled by the receiving loader.
jobs = [("seed_motion", [(motion / "g1/csv_sonic", "motion/seed_all/g1/csv_sonic"),
(motion / "metadata/seed_metadata_v004.csv", "motion/seed_all/metadata/seed_metadata_v004.csv")]),
("seed_tts", [(tts / "seed_audio", "audio/seed_tts/seed_audio"),
(tts / "metadata.jsonl", "audio/seed_tts/metadata.jsonl")])]
jobs += [(name, [(data / name, "audio/" + name)]) for name in PACKAGES]
if third_party is not None:
jobs.append((THIRD_PARTY_PACKAGE, [
(third_party / "production", "audio/hfd_third_party_multi_v5/production"),
(third_party / "training_scale_all_commands",
"audio/hfd_third_party_multi_v5/training_scale_all_commands"),
]))
return jobs
def mapped_entries(sources):
for source, destination in sources:
for path in files_under(source):
name = destination if source.is_file() else destination + "/" + path.relative_to(source).as_posix()
yield path, name
def plan_shards(name, sources, count):
"""One motion folder per sample; other files are independent samples.
Stable hash assignment avoids depending on filesystem traversal order.
Metadata JSONL/CSV remains byte-identical and is addressable via SQLite.
"""
buckets = [dict() for _ in range(count)]
total = 0
for path, logical in mapped_entries(sources):
sample_path = logical.rsplit('/', 1)[0] if name == 'seed_motion' and '/g1/csv_sonic/' in logical else logical
key = hashlib.sha256(sample_path.encode()).hexdigest()[:40]
bucket = buckets[int(key[:16], 16) % count]
field = path.name.lower() if sample_path != logical else 'payload' + (path.suffix.lower() or '.bin')
if any(ch not in 'abcdefghijklmnopqrstuvwxyz0123456789_.' for ch in field):
field = 'payload.bin'
st = path.stat()
bucket.setdefault(key, []).append((path, logical, field, st.st_size, st.st_mtime_ns))
total += st.st_size
return buckets, total
def pack_shard(dest, name, number, samples):
"""Uncompressed, contiguous WDS samples + offsets. Resume verified shards."""
rel = f'shards/{name}/{number:05d}.tar'
target = dest / rel
journal = dest / '.build' / name / f'{number:05d}.jsonl'
marker = journal.with_suffix('.done.json')
inventory = [(k, str(p), n, f, size, mt) for k, rows in sorted(samples.items()) for p, n, f, size, mt in rows]
signature = hashlib.sha256(json.dumps(inventory, ensure_ascii=False).encode()).hexdigest()
if marker.exists():
record = json.loads(marker.read_text())
if record['source_signature'] != signature:
raise RuntimeError(f'Sources changed: {rel}; choose a NEW export directory')
if not target.exists() or not journal.exists() or digest(target) != record['sha256'] or digest(journal) != record['index_sha256']:
raise RuntimeError(f'Completed shard corrupted: {rel}')
print(f'[resume] {rel}', flush=True)
return record
target.parent.mkdir(parents=True, exist_ok=True)
journal.parent.mkdir(parents=True, exist_ok=True)
partial = target.with_suffix('.tar.partial')
count = raw = 0
with tarfile.open(partial, 'w', format=tarfile.USTAR_FORMAT) as tar, journal.open('w') as index:
for key, rows in sorted(samples.items()):
fields = [row[2] for row in rows]
if len(fields) != len(set(fields)) or 'json' in fields:
raise ValueError(f'Duplicate WDS fields: {key}')
meta = {'key': key, 'package': name, 'paths': {row[2]: row[1] for row in rows}}
content = json.dumps(meta, ensure_ascii=False).encode()
info = tarfile.TarInfo(key + '.json')
info.size = len(content)
tar.addfile(info, io.BytesIO(content))
for path, logical, field, size, mtime in rows:
before = path.stat()
if path.is_symlink() or (before.st_size, before.st_mtime_ns) != (size, mtime):
raise RuntimeError(f'Source changed since planning: {path}')
info = tarfile.TarInfo(key + '.' + field)
info.size = size
offset = tar.offset + 512 # USTAR, short ASCII names: exactly one header block.
with path.open('rb') as handle:
tar.addfile(info, handle)
after = path.stat()
if (after.st_size, after.st_mtime_ns) != (size, mtime):
raise RuntimeError(f'Source changed during packaging: {path}')
index.write(json.dumps([logical, rel, offset, size, key, field], ensure_ascii=False) + '\n')
count += 1
raw += size
os.replace(partial, target)
record = {'file': rel, 'files': count, 'samples': len(samples), 'raw_bytes': raw,
'archive_bytes': target.stat().st_size, 'sha256': digest(target),
'source_signature': signature, 'index_sha256': digest(journal)}
tmp = marker.with_suffix('.partial')
tmp.write_text(json.dumps(record) + '\n')
os.replace(tmp, marker) # Only this marker constitutes a completed shard.
print(f'[done] {rel}: {len(samples):,} samples, {raw / 2**20:.1f} MiB', flush=True)
return record
def verify_package_plan(dest, name, buckets):
"""Also detect removals that would make a previously completed shard empty."""
hasher = hashlib.sha256()
for number, bucket in enumerate(buckets):
for key, rows in sorted(bucket.items()):
for path, logical, field, size, mtime in rows:
hasher.update(json.dumps([number, key, str(path), logical, field, size, mtime]).encode() + b'\n')
plan = dest / '.build' / name / 'PLAN.json'
value = {'source_signature': hasher.hexdigest(), 'shards': len(buckets)}
if plan.exists() and json.loads(plan.read_text()) != value:
raise RuntimeError(f'Package inventory changed: {name}; choose a new output directory')
plan.parent.mkdir(parents=True, exist_ok=True)
temporary = plan.with_suffix('.partial')
temporary.write_text(json.dumps(value) + '\n')
os.replace(temporary, plan)
def write_index(dest, records):
path = dest / 'index.sqlite.partial'
if path.exists():
path.unlink() # Our own unfinished index only; rebuilt from completed journals.
with sqlite3.connect(path) as conn:
conn.execute('CREATE TABLE assets(path TEXT PRIMARY KEY, shard TEXT, offset INTEGER, size INTEGER, sample_key TEXT, field TEXT)')
for record in records:
rel = Path(record['file'])
journal = dest / '.build' / rel.parent.name / (rel.stem + '.jsonl')
with journal.open() as handle:
conn.executemany('INSERT INTO assets VALUES (?,?,?,?,?,?)', (json.loads(line) for line in handle))
os.replace(path, dest / 'index.sqlite')
def completed_records(dest):
"""Rebuild the global index from every committed package, including older ones."""
records = []
for marker in sorted((dest / '.build').glob('*/*.done.json')):
record = json.loads(marker.read_text())
target = dest / record['file']
journal = marker.with_name(marker.name.replace('.done.json', '.jsonl'))
if not target.is_file() or not journal.is_file():
raise FileNotFoundError(f'Incomplete committed package marker: {marker}')
if digest(target) != record['sha256'] or digest(journal) != record['index_sha256']:
raise RuntimeError(f'Committed package checksum mismatch: {marker}')
records.append(record)
return records
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--output", type=Path, required=True, help="Project-external export directory; can resume its completed shards")
p.add_argument("--build", action="store_true", help="Write uncompressed WebDataset shards; otherwise print paths only")
p.add_argument("--shards", type=int, default=128, help="Shard count for motion, TTS and mixed waveform packages")
p.add_argument("--workers", type=int, default=4, help="CPU packing workers; no GPU usage")
p.add_argument("--part", choices=("all", "data", "code"), default="all")
p.add_argument("--motion-root", type=Path, default=ROOT.parent / "Psi0/dataset/seed_all")
p.add_argument("--tts-root", type=Path, default=ROOT.parent.parent.parent / "buwendong/user/code/qwen3-tts")
p.add_argument("--data-root", type=Path, default=ROOT.parent.parent / "videogen")
p.add_argument("--third-party-root", type=Path,
help="Parent containing production/ and training_scale_all_commands/")
p.add_argument("--only-package", choices=(*PACKAGES, "seed_motion", "seed_tts", THIRD_PARTY_PACKAGE),
help="Incrementally add/rebuild one package while preserving the global index")
a = p.parse_args()
if a.shards < 1 or a.workers < 1:
p.error('shards/workers must be positive')
out = a.output.expanduser().resolve()
protected = [ROOT, a.motion_root.resolve(), a.tts_root.resolve(), a.data_root.resolve()]
if a.third_party_root:
protected.append(a.third_party_root.resolve())
if any(out == r or out in r.parents or r in out.parents for r in protected):
raise ValueError("Output must be outside code and input data trees")
jobs = dataset_jobs(a.motion_root, a.tts_root, a.data_root, a.third_party_root)
if a.only_package:
jobs = [(name, sources) for name, sources in jobs if name == a.only_package]
if not jobs:
raise ValueError(f'{a.only_package} needs its corresponding source argument')
for name, sources in jobs if a.part != "code" else []:
for path, _ in sources:
if not path.exists():
raise FileNotFoundError(path)
print(name, [(str(src), dst) for src, dst in sources], flush=True)
if a.part != "data":
entries = list(code_entries(ROOT))
print("code files/bytes (before gzip):", len(entries), sum(f.stat().st_size for f, _ in entries))
if not a.build:
print("PLAN ONLY. Add --build to package; no HF upload is performed.")
return
if out.exists() and not (out / '.hfd-export.json').exists():
raise FileExistsError(f'Not an export owned by this tool: {out}')
out.mkdir(parents=True, exist_ok=True)
lock = (out / '.export.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
config = {'version': 2, 'shards': a.shards,
# Existing exports predate optional incremental source roots.
'roots': [str(r) for r in protected[:4]]}
ownership = out / '.hfd-export.json'
if ownership.exists() and json.loads(ownership.read_text()) != config:
raise ValueError('Export configuration changed; choose a new directory')
ownership.write_text(json.dumps(config) + '\n')
if a.part != "code":
dest = out / "hfd_data"
records = []
for name, sources in jobs:
count = a.shards if name in {'seed_motion', 'seed_tts', PACKAGES[0], THIRD_PARTY_PACKAGE} else (min(a.shards, 32) if name.endswith('hesitation_pause_v1') else 1)
print(f'[scan] {name}: preparing {count} shards', flush=True)
buckets, raw = plan_shards(name, sources, count)
verify_package_plan(dest, name, buckets)
print(f'[plan] {name}: {raw / 2**30:.3f} GiB logical bytes', flush=True)
with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:
futures = [pool.submit(pack_shard, dest, name, i, bucket) for i, bucket in enumerate(buckets) if bucket]
records.extend(f.result() for f in futures)
del buckets
# An incremental invocation must not erase the older packages from the
# shared SQLite index/manifest/checksum file.
records = completed_records(dest)
write_index(dest, records)
shutil.copyfile(ROOT / "doc/hf_audio_motion/DATA_README.md", dest / "README.md")
shutil.copyfile(ROOT / "doc/hf_audio_motion/PATH_MAP.json", dest / "PATH_MAP.json")
shutil.copyfile(ROOT / 'scripts/data/read_hf_audio_motion.py', dest / 'read_hf_audio_motion.py')
(dest / "normalization").mkdir(exist_ok=True)
shutil.copyfile(ROOT / "assets/stats/seed_all_g1_motion39_textop_stats.json",
dest / "normalization/seed_all_g1_motion39_textop_stats.json")
(dest / "MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n")
checks = ''.join(f"{r['sha256']} {r['file']}\n" for r in records)
for item in ['index.sqlite', 'PATH_MAP.json', 'read_hf_audio_motion.py', 'normalization/seed_all_g1_motion39_textop_stats.json', 'MANIFEST.json']:
checks += f'{digest(dest / item)} {item}\n'
(dest / "SHA256SUMS").write_text(checks)
if a.part != "data":
dest = out / "hfd"
if (dest / 'code/hfd_source.tar.gz').exists():
snapshot = json.loads((dest / 'SNAPSHOT.json').read_text())
if digest(dest / 'code/hfd_source.tar.gz') != snapshot['sha256']:
raise RuntimeError('Previous code snapshot checksum mismatch')
print(f'Preserved previously completed code snapshot: {dest}. Use a new output for newer code.')
return
# Keep the upstream notice in the exported copy even if locally deleted;
# do not restore or alter the user's working-tree LICENSE.
license_bytes = (ROOT / "LICENSE").read_bytes() if (ROOT / "LICENSE").exists() else subprocess.check_output(
["git", "show", "HEAD:LICENSE"], cwd=ROOT)
snapshot = {"git_head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT).decode().strip(),
"created_unix": time.time(), "source": "working tree, including uncommitted source changes",
"sanitization": "Historical commented access tokens redacted in exported copy; executable token matches rejected",
"excluded": ["environment", "credentials", "runs", "datasets", "models", "git history", "DIALOG", "historical experiment outputs"]}
record = write_archive(dest / "code/hfd_source.tar.gz", code_entries(ROOT),
{"HFD/LICENSE": license_bytes, "HFD/SNAPSHOT.json": json.dumps(snapshot, indent=2).encode()}, resume_partial=True)
shutil.copyfile(ROOT / "doc/hf_audio_motion/CODE_README.md", dest / "README.md")
(dest / "SNAPSHOT.json").write_text(json.dumps({**snapshot, **record}, indent=2) + "\n")
(dest / "SHA256SUMS").write_text(f"{record['sha256']} code/hfd_source.tar.gz\n")
print(f"Complete: {out}. Review MANIFEST/SNAPSHOT and SHA256SUMS before uploading.")
if __name__ == "__main__":
main()
|