File size: 20,015 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
#!/usr/bin/env python3
"""Offline HF handoff packaging. Default is a read-only plan; never uploads."""
from __future__ import annotations

import argparse
import concurrent.futures
import fcntl
import hashlib
import io
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
import tarfile
import time

ROOT = Path(__file__).resolve().parents[2]
PACKAGES = (
    "hfd_audio_command_continuous_office_v1",
    "hfd_control_commands_v1",
    "hfd_backchannel_commands_v2_short",
    "hfd_hesitation_pause_v1",
    "hfd_hesitation_pause_v2",
)
THIRD_PARTY_PACKAGE = "hfd_third_party_multi_v5"
SKIP_DIRS = {".git", ".venv-psi", "__pycache__", ".pytest_cache", ".cache",
             ".progress", "logs", "gt_review", "runs", "checkpoints"}
CODE_DIRS = ("src", "scripts", "tests", "assets", "backup")
CODE_FILES = ("README.md", "pyproject.toml", "uv.lock", "posttrain-full-duplex-audio-command.sh",
              "infer_render_auto.sh")


def files_under(path):
    """Never follow a link into another dataset, environment or private tree."""
    path = Path(path)
    if path.is_symlink():
        raise ValueError(f"Refusing symlink: {path}")
    if path.is_file():
        yield path
        return
    if not path.is_dir():
        raise FileNotFoundError(path)
    for directory, dirs, names in os.walk(path, followlinks=False):
        dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)
        for name in [*dirs, *names]:
            item = Path(directory) / name
            if item.is_symlink():
                raise ValueError(f"Refusing symlink: {item}")
        for name in sorted(names):
            if (name.startswith(".env") or name in {"id_rsa", "id_ed25519"}
                    or name.endswith((".pyc", ".pyo", ".key", ".pem"))):
                continue
            yield Path(directory) / name


def code_entries(root):
    roots = [root / p for p in (*CODE_DIRS, *CODE_FILES)]
    roots.append(root / "doc/hf_audio_motion")
    for base in roots:
        for path in files_under(base):
            if path.suffix.lower() in {'.pt', '.pth', '.safetensors', '.wav', '.mp4', '.pkl', '.sqlite', '.bin'}:
                continue
            yield path, "HFD/" + path.relative_to(root).as_posix()


def digest(path):
    h = hashlib.sha256()
    with path.open("rb") as handle:
        for block in iter(lambda: handle.read(8 * 1024 * 1024), b""):
            h.update(block)
    return h.hexdigest()


def safe_code_bytes(path):
    """Remove historical commented tokens in export only; reject executable secrets."""
    value = path.read_bytes()
    if re.search(rb'-----BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY-----', value):
        raise ValueError(f'Private key detected; not exporting {path}')
    token = re.compile(rb'(?<![A-Za-z0-9_])(?:hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9_-]{20,})')
    lines = []
    for line in value.splitlines(keepends=True):
        if token.search(line):
            if not line.lstrip().startswith(b'#'):
                raise ValueError(f'Possible executable credential; review {path}')
            line = token.sub(b'[REDACTED_TOKEN]', line)
        lines.append(line)
    return b''.join(lines)


def write_archive(target, entries, extras=None, resume_partial=False):
    """Atomic archive; no destructive replacement of an earlier export."""
    if target.exists():
        raise FileExistsError(target)
    target.parent.mkdir(parents=True, exist_ok=True)
    temporary = target.with_name(target.name + ".partial")
    if temporary.exists() and not resume_partial:
        raise FileExistsError(temporary)
    count = raw_bytes = 0
    with tarfile.open(temporary, "w:gz", compresslevel=1, dereference=False) as archive:
        for path, name in entries:
            if path.is_symlink() or not path.is_file():
                raise ValueError(f"Not a regular file: {path}")
            before = path.stat()
            value = safe_code_bytes(path)
            info = archive.gettarinfo(str(path), arcname=name)
            info.size = len(value)
            archive.addfile(info, io.BytesIO(value))
            after = path.stat()
            if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
                raise RuntimeError(f"Source changed during packaging: {path}")
            count += 1
            raw_bytes += len(value)
            if count % 20000 == 0:
                print(f"[pack] {target.name}: {count:,} files", flush=True)
        for name, value in (extras or {}).items():
            info = tarfile.TarInfo(name)
            info.size = len(value)
            info.mode = 0o644
            archive.addfile(info, io.BytesIO(value))
            count += 1
            raw_bytes += len(value)
    os.replace(temporary, target)
    result = {"file": target.name, "files": count, "raw_bytes": raw_bytes,
              "archive_bytes": target.stat().st_size, "sha256": digest(target)}
    print(json.dumps(result), flush=True)
    return result


def dataset_jobs(motion, tts, data, third_party=None):
    # Preserve source bytes/IDs. Absolute paths are handled by the receiving loader.
    jobs = [("seed_motion", [(motion / "g1/csv_sonic", "motion/seed_all/g1/csv_sonic"),
                             (motion / "metadata/seed_metadata_v004.csv", "motion/seed_all/metadata/seed_metadata_v004.csv")]),
            ("seed_tts", [(tts / "seed_audio", "audio/seed_tts/seed_audio"),
                          (tts / "metadata.jsonl", "audio/seed_tts/metadata.jsonl")])]
    jobs += [(name, [(data / name, "audio/" + name)]) for name in PACKAGES]
    if third_party is not None:
        jobs.append((THIRD_PARTY_PACKAGE, [
            (third_party / "production", "audio/hfd_third_party_multi_v5/production"),
            (third_party / "training_scale_all_commands",
             "audio/hfd_third_party_multi_v5/training_scale_all_commands"),
        ]))
    return jobs


def mapped_entries(sources):
    for source, destination in sources:
        for path in files_under(source):
            name = destination if source.is_file() else destination + "/" + path.relative_to(source).as_posix()
            yield path, name


def plan_shards(name, sources, count):
    """One motion folder per sample; other files are independent samples.

    Stable hash assignment avoids depending on filesystem traversal order.
    Metadata JSONL/CSV remains byte-identical and is addressable via SQLite.
    """
    buckets = [dict() for _ in range(count)]
    total = 0
    for path, logical in mapped_entries(sources):
        sample_path = logical.rsplit('/', 1)[0] if name == 'seed_motion' and '/g1/csv_sonic/' in logical else logical
        key = hashlib.sha256(sample_path.encode()).hexdigest()[:40]
        bucket = buckets[int(key[:16], 16) % count]
        field = path.name.lower() if sample_path != logical else 'payload' + (path.suffix.lower() or '.bin')
        if any(ch not in 'abcdefghijklmnopqrstuvwxyz0123456789_.' for ch in field):
            field = 'payload.bin'
        st = path.stat()
        bucket.setdefault(key, []).append((path, logical, field, st.st_size, st.st_mtime_ns))
        total += st.st_size
    return buckets, total


def pack_shard(dest, name, number, samples):
    """Uncompressed, contiguous WDS samples + offsets. Resume verified shards."""
    rel = f'shards/{name}/{number:05d}.tar'
    target = dest / rel
    journal = dest / '.build' / name / f'{number:05d}.jsonl'
    marker = journal.with_suffix('.done.json')
    inventory = [(k, str(p), n, f, size, mt) for k, rows in sorted(samples.items()) for p, n, f, size, mt in rows]
    signature = hashlib.sha256(json.dumps(inventory, ensure_ascii=False).encode()).hexdigest()
    if marker.exists():
        record = json.loads(marker.read_text())
        if record['source_signature'] != signature:
            raise RuntimeError(f'Sources changed: {rel}; choose a NEW export directory')
        if not target.exists() or not journal.exists() or digest(target) != record['sha256'] or digest(journal) != record['index_sha256']:
            raise RuntimeError(f'Completed shard corrupted: {rel}')
        print(f'[resume] {rel}', flush=True)
        return record
    target.parent.mkdir(parents=True, exist_ok=True)
    journal.parent.mkdir(parents=True, exist_ok=True)
    partial = target.with_suffix('.tar.partial')
    count = raw = 0
    with tarfile.open(partial, 'w', format=tarfile.USTAR_FORMAT) as tar, journal.open('w') as index:
        for key, rows in sorted(samples.items()):
            fields = [row[2] for row in rows]
            if len(fields) != len(set(fields)) or 'json' in fields:
                raise ValueError(f'Duplicate WDS fields: {key}')
            meta = {'key': key, 'package': name, 'paths': {row[2]: row[1] for row in rows}}
            content = json.dumps(meta, ensure_ascii=False).encode()
            info = tarfile.TarInfo(key + '.json')
            info.size = len(content)
            tar.addfile(info, io.BytesIO(content))
            for path, logical, field, size, mtime in rows:
                before = path.stat()
                if path.is_symlink() or (before.st_size, before.st_mtime_ns) != (size, mtime):
                    raise RuntimeError(f'Source changed since planning: {path}')
                info = tarfile.TarInfo(key + '.' + field)
                info.size = size
                offset = tar.offset + 512  # USTAR, short ASCII names: exactly one header block.
                with path.open('rb') as handle:
                    tar.addfile(info, handle)
                after = path.stat()
                if (after.st_size, after.st_mtime_ns) != (size, mtime):
                    raise RuntimeError(f'Source changed during packaging: {path}')
                index.write(json.dumps([logical, rel, offset, size, key, field], ensure_ascii=False) + '\n')
                count += 1
                raw += size
    os.replace(partial, target)
    record = {'file': rel, 'files': count, 'samples': len(samples), 'raw_bytes': raw,
              'archive_bytes': target.stat().st_size, 'sha256': digest(target),
              'source_signature': signature, 'index_sha256': digest(journal)}
    tmp = marker.with_suffix('.partial')
    tmp.write_text(json.dumps(record) + '\n')
    os.replace(tmp, marker)  # Only this marker constitutes a completed shard.
    print(f'[done] {rel}: {len(samples):,} samples, {raw / 2**20:.1f} MiB', flush=True)
    return record


def verify_package_plan(dest, name, buckets):
    """Also detect removals that would make a previously completed shard empty."""
    hasher = hashlib.sha256()
    for number, bucket in enumerate(buckets):
        for key, rows in sorted(bucket.items()):
            for path, logical, field, size, mtime in rows:
                hasher.update(json.dumps([number, key, str(path), logical, field, size, mtime]).encode() + b'\n')
    plan = dest / '.build' / name / 'PLAN.json'
    value = {'source_signature': hasher.hexdigest(), 'shards': len(buckets)}
    if plan.exists() and json.loads(plan.read_text()) != value:
        raise RuntimeError(f'Package inventory changed: {name}; choose a new output directory')
    plan.parent.mkdir(parents=True, exist_ok=True)
    temporary = plan.with_suffix('.partial')
    temporary.write_text(json.dumps(value) + '\n')
    os.replace(temporary, plan)


def write_index(dest, records):
    path = dest / 'index.sqlite.partial'
    if path.exists():
        path.unlink()  # Our own unfinished index only; rebuilt from completed journals.
    with sqlite3.connect(path) as conn:
        conn.execute('CREATE TABLE assets(path TEXT PRIMARY KEY, shard TEXT, offset INTEGER, size INTEGER, sample_key TEXT, field TEXT)')
        for record in records:
            rel = Path(record['file'])
            journal = dest / '.build' / rel.parent.name / (rel.stem + '.jsonl')
            with journal.open() as handle:
                conn.executemany('INSERT INTO assets VALUES (?,?,?,?,?,?)', (json.loads(line) for line in handle))
    os.replace(path, dest / 'index.sqlite')


def completed_records(dest):
    """Rebuild the global index from every committed package, including older ones."""
    records = []
    for marker in sorted((dest / '.build').glob('*/*.done.json')):
        record = json.loads(marker.read_text())
        target = dest / record['file']
        journal = marker.with_name(marker.name.replace('.done.json', '.jsonl'))
        if not target.is_file() or not journal.is_file():
            raise FileNotFoundError(f'Incomplete committed package marker: {marker}')
        if digest(target) != record['sha256'] or digest(journal) != record['index_sha256']:
            raise RuntimeError(f'Committed package checksum mismatch: {marker}')
        records.append(record)
    return records


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--output", type=Path, required=True, help="Project-external export directory; can resume its completed shards")
    p.add_argument("--build", action="store_true", help="Write uncompressed WebDataset shards; otherwise print paths only")
    p.add_argument("--shards", type=int, default=128, help="Shard count for motion, TTS and mixed waveform packages")
    p.add_argument("--workers", type=int, default=4, help="CPU packing workers; no GPU usage")
    p.add_argument("--part", choices=("all", "data", "code"), default="all")
    p.add_argument("--motion-root", type=Path, default=ROOT.parent / "Psi0/dataset/seed_all")
    p.add_argument("--tts-root", type=Path, default=ROOT.parent.parent.parent / "buwendong/user/code/qwen3-tts")
    p.add_argument("--data-root", type=Path, default=ROOT.parent.parent / "videogen")
    p.add_argument("--third-party-root", type=Path,
                   help="Parent containing production/ and training_scale_all_commands/")
    p.add_argument("--only-package", choices=(*PACKAGES, "seed_motion", "seed_tts", THIRD_PARTY_PACKAGE),
                   help="Incrementally add/rebuild one package while preserving the global index")
    a = p.parse_args()
    if a.shards < 1 or a.workers < 1:
        p.error('shards/workers must be positive')
    out = a.output.expanduser().resolve()
    protected = [ROOT, a.motion_root.resolve(), a.tts_root.resolve(), a.data_root.resolve()]
    if a.third_party_root:
        protected.append(a.third_party_root.resolve())
    if any(out == r or out in r.parents or r in out.parents for r in protected):
        raise ValueError("Output must be outside code and input data trees")
    jobs = dataset_jobs(a.motion_root, a.tts_root, a.data_root, a.third_party_root)
    if a.only_package:
        jobs = [(name, sources) for name, sources in jobs if name == a.only_package]
        if not jobs:
            raise ValueError(f'{a.only_package} needs its corresponding source argument')
    for name, sources in jobs if a.part != "code" else []:
        for path, _ in sources:
            if not path.exists():
                raise FileNotFoundError(path)
        print(name, [(str(src), dst) for src, dst in sources], flush=True)
    if a.part != "data":
        entries = list(code_entries(ROOT))
        print("code files/bytes (before gzip):", len(entries), sum(f.stat().st_size for f, _ in entries))
    if not a.build:
        print("PLAN ONLY. Add --build to package; no HF upload is performed.")
        return
    if out.exists() and not (out / '.hfd-export.json').exists():
        raise FileExistsError(f'Not an export owned by this tool: {out}')
    out.mkdir(parents=True, exist_ok=True)
    lock = (out / '.export.lock').open('a')
    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
    config = {'version': 2, 'shards': a.shards,
              # Existing exports predate optional incremental source roots.
              'roots': [str(r) for r in protected[:4]]}
    ownership = out / '.hfd-export.json'
    if ownership.exists() and json.loads(ownership.read_text()) != config:
        raise ValueError('Export configuration changed; choose a new directory')
    ownership.write_text(json.dumps(config) + '\n')
    if a.part != "code":
        dest = out / "hfd_data"
        records = []
        for name, sources in jobs:
            count = a.shards if name in {'seed_motion', 'seed_tts', PACKAGES[0], THIRD_PARTY_PACKAGE} else (min(a.shards, 32) if name.endswith('hesitation_pause_v1') else 1)
            print(f'[scan] {name}: preparing {count} shards', flush=True)
            buckets, raw = plan_shards(name, sources, count)
            verify_package_plan(dest, name, buckets)
            print(f'[plan] {name}: {raw / 2**30:.3f} GiB logical bytes', flush=True)
            with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:
                futures = [pool.submit(pack_shard, dest, name, i, bucket) for i, bucket in enumerate(buckets) if bucket]
                records.extend(f.result() for f in futures)
            del buckets
        # An incremental invocation must not erase the older packages from the
        # shared SQLite index/manifest/checksum file.
        records = completed_records(dest)
        write_index(dest, records)
        shutil.copyfile(ROOT / "doc/hf_audio_motion/DATA_README.md", dest / "README.md")
        shutil.copyfile(ROOT / "doc/hf_audio_motion/PATH_MAP.json", dest / "PATH_MAP.json")
        shutil.copyfile(ROOT / 'scripts/data/read_hf_audio_motion.py', dest / 'read_hf_audio_motion.py')
        (dest / "normalization").mkdir(exist_ok=True)
        shutil.copyfile(ROOT / "assets/stats/seed_all_g1_motion39_textop_stats.json",
                        dest / "normalization/seed_all_g1_motion39_textop_stats.json")
        (dest / "MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n")
        checks = ''.join(f"{r['sha256']}  {r['file']}\n" for r in records)
        for item in ['index.sqlite', 'PATH_MAP.json', 'read_hf_audio_motion.py', 'normalization/seed_all_g1_motion39_textop_stats.json', 'MANIFEST.json']:
            checks += f'{digest(dest / item)}  {item}\n'
        (dest / "SHA256SUMS").write_text(checks)
    if a.part != "data":
        dest = out / "hfd"
        if (dest / 'code/hfd_source.tar.gz').exists():
            snapshot = json.loads((dest / 'SNAPSHOT.json').read_text())
            if digest(dest / 'code/hfd_source.tar.gz') != snapshot['sha256']:
                raise RuntimeError('Previous code snapshot checksum mismatch')
            print(f'Preserved previously completed code snapshot: {dest}. Use a new output for newer code.')
            return
        # Keep the upstream notice in the exported copy even if locally deleted;
        # do not restore or alter the user's working-tree LICENSE.
        license_bytes = (ROOT / "LICENSE").read_bytes() if (ROOT / "LICENSE").exists() else subprocess.check_output(
            ["git", "show", "HEAD:LICENSE"], cwd=ROOT)
        snapshot = {"git_head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT).decode().strip(),
                    "created_unix": time.time(), "source": "working tree, including uncommitted source changes",
                    "sanitization": "Historical commented access tokens redacted in exported copy; executable token matches rejected",
                    "excluded": ["environment", "credentials", "runs", "datasets", "models", "git history", "DIALOG", "historical experiment outputs"]}
        record = write_archive(dest / "code/hfd_source.tar.gz", code_entries(ROOT),
                               {"HFD/LICENSE": license_bytes, "HFD/SNAPSHOT.json": json.dumps(snapshot, indent=2).encode()}, resume_partial=True)
        shutil.copyfile(ROOT / "doc/hf_audio_motion/CODE_README.md", dest / "README.md")
        (dest / "SNAPSHOT.json").write_text(json.dumps({**snapshot, **record}, indent=2) + "\n")
        (dest / "SHA256SUMS").write_text(f"{record['sha256']}  code/hfd_source.tar.gz\n")
    print(f"Complete: {out}. Review MANIFEST/SNAPSHOT and SHA256SUMS before uploading.")


if __name__ == "__main__":
    main()