Download scripts/data/hf_export_dashboard.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/hf_export_dashboard.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/hf_export_dashboard.py
-
curl -L -o hf_export_dashboard.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/hf_export_dashboard.py
14.1 kB
| #!/usr/bin/env python3 | |
| """Read-only observer for an already running HF export; never restarts/uploads. | |
| Samples only output shards, /proc and one metadata CSV. Does not rewalk source | |
| files or rehash large archives. Bound to loopback; access via DSW port forwarding. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import bisect | |
| from collections import deque | |
| import csv | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| import json | |
| import os | |
| from pathlib import Path | |
| import threading | |
| import time | |
| PACKAGES = [('seed_motion', 'Seed 动作', 128), ('seed_tts', '八音色指令', 128), | |
| ('hfd_audio_command_continuous_office_v1', '连续背景/预混音', 128), | |
| ('hfd_control_commands_v1', 'Stop/Hold', 1), | |
| ('hfd_backchannel_commands_v2_short', 'Backchannel', 1), | |
| ('hfd_hesitation_pause_v1', '犹豫/停顿 v1', 32), | |
| ('hfd_hesitation_pause_v2', '犹豫/停顿 v2', 1)] | |
| def read_json(path, default=None): | |
| try: | |
| return json.loads(path.read_text()) | |
| except (OSError, ValueError): | |
| return default | |
| def motion_keys(metadata): | |
| with metadata.open(newline='') as stream: | |
| return sorted({r['move_g1_path'].removeprefix('g1/csv/').removesuffix('.csv') for r in csv.DictReader(stream)}) | |
| def processes(export): | |
| rows = [] | |
| for item in Path('/proc').iterdir(): | |
| if not item.name.isdigit(): | |
| continue | |
| try: | |
| argv = (item / 'cmdline').read_bytes().split(b'\0') | |
| args = [a.decode(errors='replace') for a in argv if a] | |
| kind = None | |
| if any(a.endswith('/package_hf_audio_motion.py') or a == 'scripts/data/package_hf_audio_motion.py' for a in args) and str(export) in args: | |
| kind = 'pack' | |
| if 'upload' in args and any(a in {'Tinghong-Ye22/hfd_data', 'Tinghong-Ye22/hfd'} for a in args): | |
| kind = 'upload' | |
| if not kind: | |
| continue | |
| stat = (item / 'stat').read_text().rsplit(')', 1)[1].split() | |
| row = {'pid': int(item.name), 'kind': kind, 'state': stat[0], 'threads': int(stat[17])} | |
| if '--workers' in args: | |
| row['workers'] = args[args.index('--workers') + 1] | |
| # Only source directory handles are returned, never command line/env/credentials. | |
| row['source_cursor'] = None | |
| if kind == 'pack': | |
| for fd in (item / 'fd').iterdir(): | |
| try: | |
| target = os.readlink(fd) | |
| if '/g1/csv_sonic/' in target: | |
| row['source_cursor'] = target.split('/g1/csv_sonic/', 1)[1].rstrip('/') | |
| break | |
| except OSError: | |
| pass | |
| rows.append(row) | |
| except (OSError, ValueError, IndexError): | |
| continue | |
| return rows | |
| def rate(history, current_time, value): | |
| if history and history[-1][1] == 0 and value > 0: | |
| previous = history[-1] | |
| history.clear() # Do not count minutes spent waiting for this package as packing time. | |
| history.append(previous) | |
| history.append((current_time, value)) | |
| while len(history) > 2 and current_time - history[0][0] > 180: | |
| history.popleft() | |
| elapsed = current_time - history[0][0] | |
| return max(0, (value - history[0][1]) / elapsed) if elapsed >= 20 else None | |
| class Observer: | |
| def __init__(self, export, state, metadata): | |
| self.export, self.state = export, state | |
| self.keys = motion_keys(metadata) # One ~140MB CSV, not a million file stats. | |
| self.scan_seen = 0 | |
| self.histories = {name: deque() for name, _, _ in PACKAGES} | |
| self.scan_history = deque() | |
| self.latest = {} | |
| self.remote = {'status': '尚未进入上传阶段', 'repos': []} | |
| self.remote_time = 0 | |
| self.remote_history = {} | |
| self.state.mkdir(parents=True, exist_ok=True) | |
| def upload_progress(self): | |
| """Reads HF committed-file metadata, not upload buffers/network traffic.""" | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| repos = [] | |
| for repo, local in [('Tinghong-Ye22/hfd_data', 'hfd_data'), ('Tinghong-Ye22/hfd', 'hfd')]: | |
| base = self.export / local | |
| expected = {} | |
| for line in (base / 'SHA256SUMS').read_text().splitlines(): | |
| _, name = line.split(' ', 1) | |
| expected[name] = (base / name).stat().st_size | |
| # Include cards too; only count known files, ignore old remote contents. | |
| for name in ['README.md', 'SNAPSHOT.json', 'SHA256SUMS']: | |
| path = base / name | |
| if path.exists(): | |
| expected[name] = path.stat().st_size | |
| remote = {r.path: r for r in api.list_repo_tree(repo, repo_type='dataset', recursive=True) if hasattr(r, 'size')} | |
| done = sum(size for name, size in expected.items() if name in remote and remote[name].size == size) | |
| count = sum(name in remote and remote[name].size == size for name, size in expected.items()) | |
| repos.append({'repo': repo, 'visible_bytes': done, 'total_bytes': sum(expected.values()), 'visible_files': count, 'total_files': len(expected)}) | |
| self.remote = {'status': 'HF已提交可见文件(不是实时网络上传百分比;同大小不代表内容哈希已验收)', 'repos': repos} | |
| def sample(self): | |
| now = time.time() | |
| procs = processes(self.export) | |
| alive = any(p['kind'] == 'pack' for p in procs) | |
| data = self.export / 'hfd_data' | |
| rows = [] | |
| for name, label, planned in PACKAGES: | |
| shard_dir = data / 'shards' / name | |
| files = list(shard_dir.glob('*.tar')) + list(shard_dir.glob('*.partial')) | |
| written = sum(p.stat().st_size for p in files if p.exists()) | |
| markers = [read_json(p) for p in (data / '.build' / name).glob('*.done.json')] | |
| markers = [m for m in markers if m] | |
| done = len(markers) | |
| speed = rate(self.histories[name], now, written) | |
| completed_bytes = sum(m['archive_bytes'] for m in markers) | |
| # Hash-distributed shards: average size estimate, explicitly stage-local. | |
| projected = completed_bytes / done * planned if done else None | |
| eta = max(0, (projected - written) / speed) if projected and speed and done < planned else None | |
| stage = '完成' if done == planned else ('打包/校验中' if files else '等待扫描/打包') | |
| rows.append({'name': name, 'label': label, 'done': done, 'planned': planned, 'written_bytes': written, | |
| 'bytes_per_second': speed if alive else None, 'eta_seconds': eta if alive else None, | |
| 'files_done': sum(m['files'] for m in markers), 'stage': stage}) | |
| for process in procs: | |
| if process.get('source_cursor'): | |
| key = process['source_cursor'] | |
| if key in self.keys_set: | |
| self.scan_seen = max(self.scan_seen, bisect.bisect_right(self.keys, key)) | |
| if rows[0]['written_bytes']: | |
| self.scan_seen = len(self.keys) | |
| scan_speed = rate(self.scan_history, now, self.scan_seen) | |
| scan_eta = (len(self.keys) - self.scan_seen) / scan_speed if alive and scan_speed and self.scan_seen < len(self.keys) else None | |
| ready = (data / 'SHA256SUMS').exists() and (self.export / 'hfd/SHA256SUMS').exists() | |
| if ready and now - self.remote_time >= 90: | |
| self.remote_time = now | |
| # Network checks never block local sampling or page responses. | |
| def remote_update(): | |
| try: | |
| self.upload_progress() | |
| except Exception as error: | |
| self.remote = {'status': type(error).__name__ + ':HF远端状态暂不可读取,不代表上传失败', 'repos': []} | |
| threading.Thread(target=remote_update, daemon=True).start() | |
| phase = '扫描动作文件' if alive and not rows[0]['written_bytes'] else ('打包/索引中' if alive else '打包进程未运行') | |
| if ready: | |
| phase = '本地打包完成;校验或上传阶段' | |
| if any(p['kind'] == 'upload' for p in procs): | |
| phase = 'HF上传运行中' | |
| self.latest = {'updated_at': now, 'phase': phase, 'processes': procs, 'packages': rows, | |
| 'scan': {'estimated_completed_motions': self.scan_seen, 'total_motions': len(self.keys), 'eta_seconds': scan_eta}, | |
| 'remote': self.remote, 'local_ready': ready, | |
| 'note': '扫描量由当前目录与metadata排序估计;分片ETA仅限当前包。HF一次提交前远端可能始终为0,不能据此判定停滞。'} | |
| temporary = self.state / 'status.partial' | |
| temporary.write_text(json.dumps(self.latest, ensure_ascii=False)) | |
| os.replace(temporary, self.state / 'status.json') | |
| with (self.state / 'samples.jsonl').open('a') as log: | |
| log.write(json.dumps({'time': now, 'phase': phase, 'scan': self.scan_seen, | |
| 'packages': [{k: r[k] for k in ['name', 'done', 'written_bytes']} for r in rows]}) + '\n') | |
| def run(self): | |
| self.keys_set = set(self.keys) | |
| while True: | |
| try: | |
| self.sample() | |
| except Exception as error: | |
| self.latest = {'updated_at': time.time(), 'phase': '采样失败:' + type(error).__name__, 'packages': []} | |
| time.sleep(15) | |
| HTML = '''<!doctype html><html lang="zh"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <title>HFD 数据打包与上传进度</title><style>body{font:16px system-ui;background:#f5f7fa;color:#162438;max-width:1120px;margin:35px auto;padding:0 20px}h1{font-size:25px}.box{background:white;padding:20px;border-radius:12px;margin:15px 0}table{width:100%;border-collapse:collapse}td,th{text-align:left;padding:11px 8px;border-bottom:1px solid #eee}.muted{color:#65768b;font-size:14px}progress{width:100%}</style> | |
| <h1>HFD 音频→动作:打包 / 上传</h1><div class="box"><h2 id="phase">正在连接</h2><p id="scan"></p><p id="proc" class="muted"></p></div> | |
| <div class="box"><table><thead><tr><th>数据包</th><th>分片</th><th>已写</th><th>近期速度</th><th>本包剩余估计</th></tr></thead><tbody id="rows"></tbody></table></div> | |
| <div class="box"><h3>Hugging Face</h3><p id="remote"></p><div id="repos"></div></div><p id="note" class="muted"></p><p id="time" class="muted"></p> | |
| <script> | |
| const gib=x=>(x/1073741824).toFixed(2)+' GiB'; | |
| const eta=x=>x==null?'等待测速':Math.ceil(x/60)+' 分钟'; | |
| async function update(){try{let r=await fetch('api/status',{cache:'no-store'});if(!r.ok)throw Error('HTTP '+r.status);let s=await r.json(); | |
| document.getElementById('phase').textContent=Date.now()/1000-s.updated_at>90?'采样超过90秒未更新,请检查看板进程':s.phase; | |
| let a=s.scan||{};document.getElementById('scan').textContent=a.total_motions?'动作目录扫描估计:'+(a.estimated_completed_motions||0).toLocaleString()+' / '+a.total_motions.toLocaleString()+(a.estimated_completed_motions>=a.total_motions?';扫描已完成':';扫描剩余 '+eta(a.eta_seconds)):''; | |
| document.getElementById('proc').textContent=(s.processes||[]).map(p=>'PID '+p.pid+' · '+p.kind+' · '+p.threads+'线程'+(p.workers?' · 配置并发'+p.workers:'')).join(';'); | |
| const tbody=document.getElementById('rows');tbody.replaceChildren();for(let p of s.packages||[]){let tr=document.createElement('tr');for(let v of [p.label,p.done+'/'+p.planned,gib(p.written_bytes),p.bytes_per_second?(p.bytes_per_second/1048576).toFixed(1)+' MiB/s':'—',p.done===p.planned?'完成':eta(p.eta_seconds)]){let td=document.createElement('td');td.textContent=v;tr.appendChild(td)}tbody.appendChild(tr)} | |
| document.getElementById('remote').textContent=(s.remote||{}).status||'等待';const rr=document.getElementById('repos');rr.replaceChildren();for(let p of (s.remote||{}).repos||[]){let d=document.createElement('p');d.textContent=p.repo+':远端可见 '+p.visible_files+'/'+p.total_files+' 文件,'+gib(p.visible_bytes)+' / '+gib(p.total_bytes);rr.appendChild(d)} | |
| document.getElementById('note').textContent=s.note||'';document.getElementById('time').textContent='更新:'+new Date(s.updated_at*1000).toLocaleString()+' · 每15秒刷新 · 看板不读原数据内容、不消耗GPU'; | |
| }catch(e){document.getElementById('phase').textContent='连接失败:'+e.message}}update();setInterval(update,15000); | |
| </script></html>''' | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--export', type=Path, required=True) | |
| parser.add_argument('--state', type=Path, required=True) | |
| parser.add_argument('--metadata', type=Path, required=True) | |
| parser.add_argument('--port', type=int, default=8771) | |
| args = parser.parse_args() | |
| observer = Observer(args.export.resolve(), args.state.resolve(), args.metadata) | |
| threading.Thread(target=observer.run, daemon=True).start() | |
| class Handler(BaseHTTPRequestHandler): | |
| def do_GET(self): | |
| path = self.path.split('?', 1)[0] | |
| if path == '/api/status': | |
| content, mime = json.dumps(observer.latest, ensure_ascii=False).encode(), 'application/json' | |
| elif path == '/': | |
| content, mime = HTML.encode(), 'text/html' | |
| else: | |
| self.send_error(404) | |
| return | |
| self.send_response(200) | |
| self.send_header('Content-Type', mime + '; charset=utf-8') | |
| self.send_header('Content-Length', str(len(content))) | |
| self.send_header('Cache-Control', 'no-store') | |
| self.end_headers() | |
| self.wfile.write(content) | |
| def log_message(self, *_): | |
| pass | |
| print(f'Dashboard: http://127.0.0.1:{args.port}/', flush=True) | |
| ThreadingHTTPServer(('127.0.0.1', args.port), Handler).serve_forever() | |
| if __name__ == '__main__': | |
| main() | |