code-HFD / scripts /data /hf_export_dashboard.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
14.1 kB
#!/usr/bin/env python3
"""Read-only observer for an already running HF export; never restarts/uploads.
Samples only output shards, /proc and one metadata CSV. Does not rewalk source
files or rehash large archives. Bound to loopback; access via DSW port forwarding.
"""
from __future__ import annotations
import argparse
import bisect
from collections import deque
import csv
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import threading
import time
PACKAGES = [('seed_motion', 'Seed 动作', 128), ('seed_tts', '八音色指令', 128),
('hfd_audio_command_continuous_office_v1', '连续背景/预混音', 128),
('hfd_control_commands_v1', 'Stop/Hold', 1),
('hfd_backchannel_commands_v2_short', 'Backchannel', 1),
('hfd_hesitation_pause_v1', '犹豫/停顿 v1', 32),
('hfd_hesitation_pause_v2', '犹豫/停顿 v2', 1)]
def read_json(path, default=None):
try:
return json.loads(path.read_text())
except (OSError, ValueError):
return default
def motion_keys(metadata):
with metadata.open(newline='') as stream:
return sorted({r['move_g1_path'].removeprefix('g1/csv/').removesuffix('.csv') for r in csv.DictReader(stream)})
def processes(export):
rows = []
for item in Path('/proc').iterdir():
if not item.name.isdigit():
continue
try:
argv = (item / 'cmdline').read_bytes().split(b'\0')
args = [a.decode(errors='replace') for a in argv if a]
kind = None
if any(a.endswith('/package_hf_audio_motion.py') or a == 'scripts/data/package_hf_audio_motion.py' for a in args) and str(export) in args:
kind = 'pack'
if 'upload' in args and any(a in {'Tinghong-Ye22/hfd_data', 'Tinghong-Ye22/hfd'} for a in args):
kind = 'upload'
if not kind:
continue
stat = (item / 'stat').read_text().rsplit(')', 1)[1].split()
row = {'pid': int(item.name), 'kind': kind, 'state': stat[0], 'threads': int(stat[17])}
if '--workers' in args:
row['workers'] = args[args.index('--workers') + 1]
# Only source directory handles are returned, never command line/env/credentials.
row['source_cursor'] = None
if kind == 'pack':
for fd in (item / 'fd').iterdir():
try:
target = os.readlink(fd)
if '/g1/csv_sonic/' in target:
row['source_cursor'] = target.split('/g1/csv_sonic/', 1)[1].rstrip('/')
break
except OSError:
pass
rows.append(row)
except (OSError, ValueError, IndexError):
continue
return rows
def rate(history, current_time, value):
if history and history[-1][1] == 0 and value > 0:
previous = history[-1]
history.clear() # Do not count minutes spent waiting for this package as packing time.
history.append(previous)
history.append((current_time, value))
while len(history) > 2 and current_time - history[0][0] > 180:
history.popleft()
elapsed = current_time - history[0][0]
return max(0, (value - history[0][1]) / elapsed) if elapsed >= 20 else None
class Observer:
def __init__(self, export, state, metadata):
self.export, self.state = export, state
self.keys = motion_keys(metadata) # One ~140MB CSV, not a million file stats.
self.scan_seen = 0
self.histories = {name: deque() for name, _, _ in PACKAGES}
self.scan_history = deque()
self.latest = {}
self.remote = {'status': '尚未进入上传阶段', 'repos': []}
self.remote_time = 0
self.remote_history = {}
self.state.mkdir(parents=True, exist_ok=True)
def upload_progress(self):
"""Reads HF committed-file metadata, not upload buffers/network traffic."""
from huggingface_hub import HfApi
api = HfApi()
repos = []
for repo, local in [('Tinghong-Ye22/hfd_data', 'hfd_data'), ('Tinghong-Ye22/hfd', 'hfd')]:
base = self.export / local
expected = {}
for line in (base / 'SHA256SUMS').read_text().splitlines():
_, name = line.split(' ', 1)
expected[name] = (base / name).stat().st_size
# Include cards too; only count known files, ignore old remote contents.
for name in ['README.md', 'SNAPSHOT.json', 'SHA256SUMS']:
path = base / name
if path.exists():
expected[name] = path.stat().st_size
remote = {r.path: r for r in api.list_repo_tree(repo, repo_type='dataset', recursive=True) if hasattr(r, 'size')}
done = sum(size for name, size in expected.items() if name in remote and remote[name].size == size)
count = sum(name in remote and remote[name].size == size for name, size in expected.items())
repos.append({'repo': repo, 'visible_bytes': done, 'total_bytes': sum(expected.values()), 'visible_files': count, 'total_files': len(expected)})
self.remote = {'status': 'HF已提交可见文件(不是实时网络上传百分比;同大小不代表内容哈希已验收)', 'repos': repos}
def sample(self):
now = time.time()
procs = processes(self.export)
alive = any(p['kind'] == 'pack' for p in procs)
data = self.export / 'hfd_data'
rows = []
for name, label, planned in PACKAGES:
shard_dir = data / 'shards' / name
files = list(shard_dir.glob('*.tar')) + list(shard_dir.glob('*.partial'))
written = sum(p.stat().st_size for p in files if p.exists())
markers = [read_json(p) for p in (data / '.build' / name).glob('*.done.json')]
markers = [m for m in markers if m]
done = len(markers)
speed = rate(self.histories[name], now, written)
completed_bytes = sum(m['archive_bytes'] for m in markers)
# Hash-distributed shards: average size estimate, explicitly stage-local.
projected = completed_bytes / done * planned if done else None
eta = max(0, (projected - written) / speed) if projected and speed and done < planned else None
stage = '完成' if done == planned else ('打包/校验中' if files else '等待扫描/打包')
rows.append({'name': name, 'label': label, 'done': done, 'planned': planned, 'written_bytes': written,
'bytes_per_second': speed if alive else None, 'eta_seconds': eta if alive else None,
'files_done': sum(m['files'] for m in markers), 'stage': stage})
for process in procs:
if process.get('source_cursor'):
key = process['source_cursor']
if key in self.keys_set:
self.scan_seen = max(self.scan_seen, bisect.bisect_right(self.keys, key))
if rows[0]['written_bytes']:
self.scan_seen = len(self.keys)
scan_speed = rate(self.scan_history, now, self.scan_seen)
scan_eta = (len(self.keys) - self.scan_seen) / scan_speed if alive and scan_speed and self.scan_seen < len(self.keys) else None
ready = (data / 'SHA256SUMS').exists() and (self.export / 'hfd/SHA256SUMS').exists()
if ready and now - self.remote_time >= 90:
self.remote_time = now
# Network checks never block local sampling or page responses.
def remote_update():
try:
self.upload_progress()
except Exception as error:
self.remote = {'status': type(error).__name__ + ':HF远端状态暂不可读取,不代表上传失败', 'repos': []}
threading.Thread(target=remote_update, daemon=True).start()
phase = '扫描动作文件' if alive and not rows[0]['written_bytes'] else ('打包/索引中' if alive else '打包进程未运行')
if ready:
phase = '本地打包完成;校验或上传阶段'
if any(p['kind'] == 'upload' for p in procs):
phase = 'HF上传运行中'
self.latest = {'updated_at': now, 'phase': phase, 'processes': procs, 'packages': rows,
'scan': {'estimated_completed_motions': self.scan_seen, 'total_motions': len(self.keys), 'eta_seconds': scan_eta},
'remote': self.remote, 'local_ready': ready,
'note': '扫描量由当前目录与metadata排序估计;分片ETA仅限当前包。HF一次提交前远端可能始终为0,不能据此判定停滞。'}
temporary = self.state / 'status.partial'
temporary.write_text(json.dumps(self.latest, ensure_ascii=False))
os.replace(temporary, self.state / 'status.json')
with (self.state / 'samples.jsonl').open('a') as log:
log.write(json.dumps({'time': now, 'phase': phase, 'scan': self.scan_seen,
'packages': [{k: r[k] for k in ['name', 'done', 'written_bytes']} for r in rows]}) + '\n')
def run(self):
self.keys_set = set(self.keys)
while True:
try:
self.sample()
except Exception as error:
self.latest = {'updated_at': time.time(), 'phase': '采样失败:' + type(error).__name__, 'packages': []}
time.sleep(15)
HTML = '''<!doctype html><html lang="zh"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>HFD 数据打包与上传进度</title><style>body{font:16px system-ui;background:#f5f7fa;color:#162438;max-width:1120px;margin:35px auto;padding:0 20px}h1{font-size:25px}.box{background:white;padding:20px;border-radius:12px;margin:15px 0}table{width:100%;border-collapse:collapse}td,th{text-align:left;padding:11px 8px;border-bottom:1px solid #eee}.muted{color:#65768b;font-size:14px}progress{width:100%}</style>
<h1>HFD 音频→动作:打包 / 上传</h1><div class="box"><h2 id="phase">正在连接</h2><p id="scan"></p><p id="proc" class="muted"></p></div>
<div class="box"><table><thead><tr><th>数据包</th><th>分片</th><th>已写</th><th>近期速度</th><th>本包剩余估计</th></tr></thead><tbody id="rows"></tbody></table></div>
<div class="box"><h3>Hugging Face</h3><p id="remote"></p><div id="repos"></div></div><p id="note" class="muted"></p><p id="time" class="muted"></p>
<script>
const gib=x=>(x/1073741824).toFixed(2)+' GiB';
const eta=x=>x==null?'等待测速':Math.ceil(x/60)+' 分钟';
async function update(){try{let r=await fetch('api/status',{cache:'no-store'});if(!r.ok)throw Error('HTTP '+r.status);let s=await r.json();
document.getElementById('phase').textContent=Date.now()/1000-s.updated_at>90?'采样超过90秒未更新,请检查看板进程':s.phase;
let a=s.scan||{};document.getElementById('scan').textContent=a.total_motions?'动作目录扫描估计:'+(a.estimated_completed_motions||0).toLocaleString()+' / '+a.total_motions.toLocaleString()+(a.estimated_completed_motions>=a.total_motions?';扫描已完成':';扫描剩余 '+eta(a.eta_seconds)):'';
document.getElementById('proc').textContent=(s.processes||[]).map(p=>'PID '+p.pid+' · '+p.kind+' · '+p.threads+'线程'+(p.workers?' · 配置并发'+p.workers:'')).join(';');
const tbody=document.getElementById('rows');tbody.replaceChildren();for(let p of s.packages||[]){let tr=document.createElement('tr');for(let v of [p.label,p.done+'/'+p.planned,gib(p.written_bytes),p.bytes_per_second?(p.bytes_per_second/1048576).toFixed(1)+' MiB/s':'—',p.done===p.planned?'完成':eta(p.eta_seconds)]){let td=document.createElement('td');td.textContent=v;tr.appendChild(td)}tbody.appendChild(tr)}
document.getElementById('remote').textContent=(s.remote||{}).status||'等待';const rr=document.getElementById('repos');rr.replaceChildren();for(let p of (s.remote||{}).repos||[]){let d=document.createElement('p');d.textContent=p.repo+':远端可见 '+p.visible_files+'/'+p.total_files+' 文件,'+gib(p.visible_bytes)+' / '+gib(p.total_bytes);rr.appendChild(d)}
document.getElementById('note').textContent=s.note||'';document.getElementById('time').textContent='更新:'+new Date(s.updated_at*1000).toLocaleString()+' · 每15秒刷新 · 看板不读原数据内容、不消耗GPU';
}catch(e){document.getElementById('phase').textContent='连接失败:'+e.message}}update();setInterval(update,15000);
</script></html>'''
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--export', type=Path, required=True)
parser.add_argument('--state', type=Path, required=True)
parser.add_argument('--metadata', type=Path, required=True)
parser.add_argument('--port', type=int, default=8771)
args = parser.parse_args()
observer = Observer(args.export.resolve(), args.state.resolve(), args.metadata)
threading.Thread(target=observer.run, daemon=True).start()
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
path = self.path.split('?', 1)[0]
if path == '/api/status':
content, mime = json.dumps(observer.latest, ensure_ascii=False).encode(), 'application/json'
elif path == '/':
content, mime = HTML.encode(), 'text/html'
else:
self.send_error(404)
return
self.send_response(200)
self.send_header('Content-Type', mime + '; charset=utf-8')
self.send_header('Content-Length', str(len(content)))
self.send_header('Cache-Control', 'no-store')
self.end_headers()
self.wfile.write(content)
def log_message(self, *_):
pass
print(f'Dashboard: http://127.0.0.1:{args.port}/', flush=True)
ThreadingHTTPServer(('127.0.0.1', args.port), Handler).serve_forever()
if __name__ == '__main__':
main()