#!/usr/bin/env python3 """Read-only observer for an already running HF export; never restarts/uploads. Samples only output shards, /proc and one metadata CSV. Does not rewalk source files or rehash large archives. Bound to loopback; access via DSW port forwarding. """ from __future__ import annotations import argparse import bisect from collections import deque import csv from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import threading import time PACKAGES = [('seed_motion', 'Seed 动作', 128), ('seed_tts', '八音色指令', 128), ('hfd_audio_command_continuous_office_v1', '连续背景/预混音', 128), ('hfd_control_commands_v1', 'Stop/Hold', 1), ('hfd_backchannel_commands_v2_short', 'Backchannel', 1), ('hfd_hesitation_pause_v1', '犹豫/停顿 v1', 32), ('hfd_hesitation_pause_v2', '犹豫/停顿 v2', 1)] def read_json(path, default=None): try: return json.loads(path.read_text()) except (OSError, ValueError): return default def motion_keys(metadata): with metadata.open(newline='') as stream: return sorted({r['move_g1_path'].removeprefix('g1/csv/').removesuffix('.csv') for r in csv.DictReader(stream)}) def processes(export): rows = [] for item in Path('/proc').iterdir(): if not item.name.isdigit(): continue try: argv = (item / 'cmdline').read_bytes().split(b'\0') args = [a.decode(errors='replace') for a in argv if a] kind = None if any(a.endswith('/package_hf_audio_motion.py') or a == 'scripts/data/package_hf_audio_motion.py' for a in args) and str(export) in args: kind = 'pack' if 'upload' in args and any(a in {'Tinghong-Ye22/hfd_data', 'Tinghong-Ye22/hfd'} for a in args): kind = 'upload' if not kind: continue stat = (item / 'stat').read_text().rsplit(')', 1)[1].split() row = {'pid': int(item.name), 'kind': kind, 'state': stat[0], 'threads': int(stat[17])} if '--workers' in args: row['workers'] = args[args.index('--workers') + 1] # Only source directory handles are returned, never command line/env/credentials. row['source_cursor'] = None if kind == 'pack': for fd in (item / 'fd').iterdir(): try: target = os.readlink(fd) if '/g1/csv_sonic/' in target: row['source_cursor'] = target.split('/g1/csv_sonic/', 1)[1].rstrip('/') break except OSError: pass rows.append(row) except (OSError, ValueError, IndexError): continue return rows def rate(history, current_time, value): if history and history[-1][1] == 0 and value > 0: previous = history[-1] history.clear() # Do not count minutes spent waiting for this package as packing time. history.append(previous) history.append((current_time, value)) while len(history) > 2 and current_time - history[0][0] > 180: history.popleft() elapsed = current_time - history[0][0] return max(0, (value - history[0][1]) / elapsed) if elapsed >= 20 else None class Observer: def __init__(self, export, state, metadata): self.export, self.state = export, state self.keys = motion_keys(metadata) # One ~140MB CSV, not a million file stats. self.scan_seen = 0 self.histories = {name: deque() for name, _, _ in PACKAGES} self.scan_history = deque() self.latest = {} self.remote = {'status': '尚未进入上传阶段', 'repos': []} self.remote_time = 0 self.remote_history = {} self.state.mkdir(parents=True, exist_ok=True) def upload_progress(self): """Reads HF committed-file metadata, not upload buffers/network traffic.""" from huggingface_hub import HfApi api = HfApi() repos = [] for repo, local in [('Tinghong-Ye22/hfd_data', 'hfd_data'), ('Tinghong-Ye22/hfd', 'hfd')]: base = self.export / local expected = {} for line in (base / 'SHA256SUMS').read_text().splitlines(): _, name = line.split(' ', 1) expected[name] = (base / name).stat().st_size # Include cards too; only count known files, ignore old remote contents. for name in ['README.md', 'SNAPSHOT.json', 'SHA256SUMS']: path = base / name if path.exists(): expected[name] = path.stat().st_size remote = {r.path: r for r in api.list_repo_tree(repo, repo_type='dataset', recursive=True) if hasattr(r, 'size')} done = sum(size for name, size in expected.items() if name in remote and remote[name].size == size) count = sum(name in remote and remote[name].size == size for name, size in expected.items()) repos.append({'repo': repo, 'visible_bytes': done, 'total_bytes': sum(expected.values()), 'visible_files': count, 'total_files': len(expected)}) self.remote = {'status': 'HF已提交可见文件(不是实时网络上传百分比;同大小不代表内容哈希已验收)', 'repos': repos} def sample(self): now = time.time() procs = processes(self.export) alive = any(p['kind'] == 'pack' for p in procs) data = self.export / 'hfd_data' rows = [] for name, label, planned in PACKAGES: shard_dir = data / 'shards' / name files = list(shard_dir.glob('*.tar')) + list(shard_dir.glob('*.partial')) written = sum(p.stat().st_size for p in files if p.exists()) markers = [read_json(p) for p in (data / '.build' / name).glob('*.done.json')] markers = [m for m in markers if m] done = len(markers) speed = rate(self.histories[name], now, written) completed_bytes = sum(m['archive_bytes'] for m in markers) # Hash-distributed shards: average size estimate, explicitly stage-local. projected = completed_bytes / done * planned if done else None eta = max(0, (projected - written) / speed) if projected and speed and done < planned else None stage = '完成' if done == planned else ('打包/校验中' if files else '等待扫描/打包') rows.append({'name': name, 'label': label, 'done': done, 'planned': planned, 'written_bytes': written, 'bytes_per_second': speed if alive else None, 'eta_seconds': eta if alive else None, 'files_done': sum(m['files'] for m in markers), 'stage': stage}) for process in procs: if process.get('source_cursor'): key = process['source_cursor'] if key in self.keys_set: self.scan_seen = max(self.scan_seen, bisect.bisect_right(self.keys, key)) if rows[0]['written_bytes']: self.scan_seen = len(self.keys) scan_speed = rate(self.scan_history, now, self.scan_seen) scan_eta = (len(self.keys) - self.scan_seen) / scan_speed if alive and scan_speed and self.scan_seen < len(self.keys) else None ready = (data / 'SHA256SUMS').exists() and (self.export / 'hfd/SHA256SUMS').exists() if ready and now - self.remote_time >= 90: self.remote_time = now # Network checks never block local sampling or page responses. def remote_update(): try: self.upload_progress() except Exception as error: self.remote = {'status': type(error).__name__ + ':HF远端状态暂不可读取,不代表上传失败', 'repos': []} threading.Thread(target=remote_update, daemon=True).start() phase = '扫描动作文件' if alive and not rows[0]['written_bytes'] else ('打包/索引中' if alive else '打包进程未运行') if ready: phase = '本地打包完成;校验或上传阶段' if any(p['kind'] == 'upload' for p in procs): phase = 'HF上传运行中' self.latest = {'updated_at': now, 'phase': phase, 'processes': procs, 'packages': rows, 'scan': {'estimated_completed_motions': self.scan_seen, 'total_motions': len(self.keys), 'eta_seconds': scan_eta}, 'remote': self.remote, 'local_ready': ready, 'note': '扫描量由当前目录与metadata排序估计;分片ETA仅限当前包。HF一次提交前远端可能始终为0,不能据此判定停滞。'} temporary = self.state / 'status.partial' temporary.write_text(json.dumps(self.latest, ensure_ascii=False)) os.replace(temporary, self.state / 'status.json') with (self.state / 'samples.jsonl').open('a') as log: log.write(json.dumps({'time': now, 'phase': phase, 'scan': self.scan_seen, 'packages': [{k: r[k] for k in ['name', 'done', 'written_bytes']} for r in rows]}) + '\n') def run(self): self.keys_set = set(self.keys) while True: try: self.sample() except Exception as error: self.latest = {'updated_at': time.time(), 'phase': '采样失败:' + type(error).__name__, 'packages': []} time.sleep(15) HTML = '''
| 数据包 | 分片 | 已写 | 近期速度 | 本包剩余估计 |
|---|