Download scripts/data/review_dashboard/server.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 21 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/review_dashboard/server.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/review_dashboard/server.py
-
curl -L -o server.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/review_dashboard/server.py
21 kB
| """Local VQA review UI. Dataset files are read-only; index/notes live outside git. | |
| Usage: python server.py --build (once), then python server.py | |
| No GPU, model calls, or training-manifest mutations. | |
| """ | |
| import argparse | |
| from concurrent.futures import ThreadPoolExecutor | |
| from contextlib import closing | |
| import hashlib | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| import json | |
| import mimetypes | |
| import os | |
| from pathlib import Path | |
| import re | |
| import sqlite3 | |
| import time | |
| from urllib.parse import parse_qs, urlsplit | |
| HERE = Path(__file__).resolve().parent | |
| WORKSPACE = HERE.parents[4] | |
| DEFAULT_DATA = WORKSPACE / 'videogen' | |
| DEFAULT_STATE = WORKSPACE / 'hfd_experiments/vqa_review_dashboard' | |
| SOURCES = { | |
| 'gqa': ('GQA 图片', 'image', 'hfd_gqa_v1', 'hfd_gqa_8voice_v1'), | |
| 'vqav2': ('VQAv2 图片', 'image', 'hfd_swa_vqav2_v1', 'hfd_vqav2_8voice_v1'), | |
| 'long_video': ('LLaVA 长视频 · 实时问答', 'video', None, 'hfd_visual_qa/components/long_video'), | |
| 'legacy_video': ('历史短视频 · 非实时准入', 'video', 'hfd_video_vqa_50k_v1', None), | |
| } | |
| def read(path, default=None): | |
| if not path or not Path(path).is_file(): | |
| return default | |
| with Path(path).open() as f: | |
| return json.load(f) | |
| def rows(path): | |
| obj = read(path, []) | |
| return obj if isinstance(obj, list) else obj.get('assets', obj.get('rows', [])) | |
| def digest(path): | |
| h = hashlib.sha256() | |
| with Path(path).open('rb') as f: | |
| for block in iter(lambda: f.read(4 * 1024 * 1024), b''): | |
| h.update(block) | |
| return h.hexdigest() | |
| def asset_id(source, row): | |
| value = str(row.get('asset_id', row.get('id', ''))) | |
| if not value: | |
| raise ValueError('Row without asset identity') | |
| return 'gqa_' + value if source == 'gqa' and not value.startswith('gqa_') else value | |
| def classify(source, admitted, review, row): | |
| if source == 'legacy_video': | |
| return 'legacy', '历史审核口径;不代表通过当前实时问答准入' | |
| if admitted: | |
| return 'admitted', '最终发布清单已收录,验收哈希匹配;不等于人工审核通过' | |
| if row.get('rejected'): | |
| return 'excluded', str(row['rejected']) | |
| if review.get('accepted') is False: | |
| return 'excluded', str(review.get('reason') or '视觉/问答审核未通过;点击查看原始依据') | |
| if review.get('accepted') is True: | |
| return 'not_admitted', '视觉审核通过,但未收录最终八音色发布清单;具体原因以回执为准' | |
| return 'pending', '没有找到视觉审核结果,不能视为通过' | |
| def allowed_path(value, data): | |
| if not value: | |
| return None | |
| p = Path(value).resolve() | |
| if not p.is_relative_to(data.resolve()): | |
| return None | |
| return p | |
| def build(data, state): | |
| """Explicit offline rebuild, atomic replacement; browser never scans big manifests.""" | |
| state.mkdir(parents=True, exist_ok=True) | |
| tmp = state / 'catalog.building.sqlite' | |
| if tmp.exists(): | |
| raise RuntimeError(f'Unfinished/concurrent build: {tmp}; inspect before removing') | |
| started = time.time() | |
| db = sqlite3.connect(tmp) | |
| db.executescript(''' | |
| CREATE TABLE items (id TEXT PRIMARY KEY, source TEXT, kind TEXT, status TEXT, | |
| split TEXT, question TEXT, answer TEXT, duration REAL, voice_count INTEGER, | |
| review_pass INTEGER, media TEXT, reason TEXT, detail TEXT); | |
| CREATE TABLE media (id TEXT PRIMARY KEY, path TEXT NOT NULL); | |
| CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT); | |
| ''') | |
| def media(value): | |
| p = allowed_path(value, data) | |
| if not p or p.suffix.lower() not in {'.jpg', '.jpeg', '.png', '.webp', '.mp4', '.wav', '.mp3'}: | |
| return None | |
| key = hashlib.sha256(str(p).encode()).hexdigest() | |
| db.execute('INSERT OR IGNORE INTO media VALUES (?,?)', (key, str(p))) | |
| return key | |
| manifests = {} | |
| for source, (label, kind, origin, release) in SOURCES.items(): | |
| print('Indexing', source, flush=True) | |
| pool, variants, representatives = {}, {}, {} | |
| if origin: | |
| root = data / origin | |
| # Union by QA identity, not voices; retain technically rejected candidates too. | |
| for name in ['candidates.json', 'image_checked.json', 'image_rejections.json', 'overlay_assets.json']: | |
| for row in rows(root / name): | |
| aid = asset_id(source, row) | |
| pool[aid] = {**pool.get(aid, {}), **row} | |
| else: | |
| root = data / 'hfd_visual_qa/review/long_video' | |
| pool = {asset_id(source, r): r for r in rows(root / 'jobs.json')} | |
| if release: | |
| manifest = data / release / 'overlay_assets.json' | |
| acceptance = read(manifest.parent / 'admission/acceptance.json', {}) | |
| sha = digest(manifest) | |
| if acceptance.get('passed') is not True or acceptance.get('manifest_sha256') != sha: | |
| raise ValueError(f'Acceptance mismatch: {manifest}') | |
| manifests[source] = {'path': str(manifest), 'sha256': sha} | |
| for row in rows(manifest): | |
| aid = asset_id(source, row) | |
| if aid not in representatives: | |
| representatives[aid] = row | |
| variants.setdefault(aid, []).append({k: row[k] for k in ( | |
| 'voice', 'question', 'answer', 'question_audio_path', 'asr_receipt', | |
| 'question_translation_receipt', 'answer_translation_receipt') if k in row}) | |
| for aid, row in representatives.items(): | |
| pool[aid] = {**pool.get(aid, {}), **row} | |
| for aid, group in variants.items(): | |
| if len(group) != 8 or len({r['voice'] for r in group}) != 8: | |
| raise ValueError(f'Not exactly eight unique voices: {aid}') | |
| def load_review(pair): | |
| aid, row = pair | |
| p = row.get('visual_receipt') | |
| if not p: | |
| if source == 'long_video': | |
| p = root / 'reviews' / f'{aid}.json' | |
| elif kind == 'image': | |
| rid = aid.removeprefix('gqa_') if source == 'gqa' else aid | |
| p = root / 'visual_review' / f'{rid}.json' | |
| else: | |
| p = root / 'video_review_v2' / f'{aid}.json' | |
| p = allowed_path(p, data) | |
| return aid, row, str(p) if p else None, read(p, {}) | |
| with ThreadPoolExecutor(max_workers=8) as executor: | |
| for aid, row, receipt, review in executor.map(load_review, pool.items()): | |
| status, reason = classify(source, aid in variants, review, row) | |
| voices = variants.get(aid, []) | |
| if not voices and row.get('question_audio_path'): | |
| voices = [{k: row[k] for k in ('voice', 'question', 'answer', 'question_audio_path', 'asr_receipt') if k in row}] | |
| voices = sorted(voices, key=lambda r: r.get('voice', '')) | |
| for v in voices: | |
| v['audio'] = media(v.get('question_audio_path')) | |
| visual = media(row.get('video_path') or row.get('image_path')) | |
| detail = dict(asset_id=aid, source_label=label, voices=voices, receipt=receipt, | |
| media_path=row.get('video_path') or row.get('image_path'), | |
| parent_video_id=row.get('parent_video_id'), image_id=row.get('image_id', row.get('imageId')), | |
| live_qa_windows=row.get('live_qa_windows', review.get('live_qa_windows', [])), | |
| upstream_annotation=row.get('annotation_path'), | |
| provenance=row.get('annotation_provenance', '原始数据集问答;机器筛查,不代表人工复核'), | |
| visual_review=review, original_status=row.get('status'), | |
| manifest=manifests.get(source), original_candidate=row if not visual else None) | |
| db.execute('INSERT INTO items VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', ( | |
| source + ':' + aid, source, kind, status, row.get('split', row.get('split_proposal', 'unknown')), | |
| row.get('question', ''), row.get('answer', ''), row.get('duration_seconds'), | |
| len(variants.get(aid, [])), None if 'accepted' not in review else int(review['accepted']), | |
| visual, reason, json.dumps(detail, ensure_ascii=False))) | |
| db.commit() | |
| print(source, len(pool), 'candidates;', len(variants), 'published', flush=True) | |
| gt = [] | |
| for p in sorted((data / 'hfd_visual_qa/components/long_video/admission/acceptance').glob('*/unified_multimodal_gt_with_audio.mp4')): | |
| gt.append(dict(name=p.parent.name, media=media(p), note='GT 数据拼接;回答音频为 flite 审核占位,不代表正式模型语音质量')) | |
| db.executescript('''CREATE INDEX filtering ON items(source,status,split); | |
| CREATE INDEX statuses ON items(status,kind); CREATE INDEX durations ON items(duration); | |
| CREATE INDEX review_result ON items(review_pass);''') | |
| metadata = dict(built_at=time.time(), build_seconds=time.time()-started, | |
| sources={k:v[0] for k,v in SOURCES.items()}, manifests=manifests, gt=gt, | |
| count_unit='独立问答条目;八音色合并展示,不按八倍计数;同一画面可对应多条问答') | |
| for k, v in metadata.items(): | |
| db.execute('INSERT INTO metadata VALUES (?,?)', (k, json.dumps(v, ensure_ascii=False))) | |
| db.commit() | |
| if db.execute('PRAGMA quick_check').fetchone()[0] != 'ok': | |
| raise RuntimeError('Index integrity check failed') | |
| db.close() | |
| os.replace(tmp, state / 'catalog.sqlite') | |
| print('Built', state / 'catalog.sqlite', 'in', round(time.time()-started, 1), 'seconds', flush=True) | |
| def byte_range(value, size): | |
| if not value: | |
| return 0, size - 1, False | |
| m = re.fullmatch(r'bytes=(\d*)-(\d*)', value) | |
| if not m or not any(m.groups()) or size == 0: | |
| raise ValueError('Invalid range') | |
| a, b = m.groups() | |
| start, end = (int(a), min(int(b) if b else size-1, size-1)) if a else (max(0, size-int(b)), size-1) | |
| if start > end or start >= size: | |
| raise ValueError('Unsatisfiable range') | |
| return start, end, True | |
| class Catalog: | |
| def __init__(self, state, data): | |
| self.state, self.data = state, data | |
| if not (state / 'catalog.sqlite').exists(): | |
| raise RuntimeError('Build index first: --build') | |
| with closing(sqlite3.connect(state / 'notes.sqlite')) as db: | |
| db.execute('CREATE TABLE IF NOT EXISTS notes(id TEXT PRIMARY KEY, verdict TEXT, note TEXT, updated REAL)') | |
| db.commit() | |
| def connect(self): | |
| db = sqlite3.connect(f'file:{self.state / "catalog.sqlite"}?mode=ro', uri=True) | |
| db.row_factory = sqlite3.Row | |
| db.execute('ATTACH DATABASE ? AS human', (str(self.state / 'notes.sqlite'),)) | |
| return db | |
| def stats(self): | |
| with closing(self.connect()) as db: | |
| meta = {r['key']:json.loads(r['value']) for r in db.execute('SELECT * FROM metadata')} | |
| meta['counts'] = [dict(r) for r in db.execute('SELECT source,status,kind,count(*) AS n FROM items GROUP BY source,status,kind')] | |
| meta['human_counts'] = [dict(r) for r in db.execute('SELECT verdict,count(*) AS n FROM human.notes GROUP BY verdict')] | |
| return meta | |
| def listing(self, args): | |
| clauses, values = [], [] | |
| for name in ('source', 'kind', 'split', 'status'): | |
| if args.get(name): | |
| clauses.append(f'i.{name}=?'); values.append(args[name]) | |
| scope = args.get('scope', 'admitted') | |
| if scope == 'admitted': clauses.append("i.status='admitted'") | |
| elif scope == 'candidates': clauses.append("i.status!='legacy'") | |
| elif scope == 'excluded': clauses.append("i.status IN ('excluded','not_admitted')") | |
| elif scope == 'pending': clauses.append("i.status='pending'") | |
| elif scope == 'legacy': clauses.append("i.status='legacy'") | |
| elif scope != 'all': raise ValueError('Invalid scope') | |
| if args.get('q'): | |
| clauses.append('(i.question LIKE ? OR i.answer LIKE ? OR i.id LIKE ?)') | |
| values += ['%' + args['q'][:200] + '%'] * 3 | |
| if args.get('human') == 'unreviewed': clauses.append('h.id IS NULL') | |
| elif args.get('human'): | |
| clauses.append('h.verdict=?'); values.append(args['human']) | |
| for name, op in [('min_duration', '>='), ('max_duration', '<=')]: | |
| if args.get(name): | |
| clauses.append(f'i.duration {op} ?'); values.append(float(args[name])) | |
| if args.get('review') in ('yes', 'no'): | |
| clauses.append('i.review_pass=?'); values.append(int(args['review'] == 'yes')) | |
| limit = min(48, max(1, int(args.get('limit', 24)))) | |
| page = max(0, int(args.get('page', 0))) | |
| where = ' WHERE ' + ' AND '.join(clauses) if clauses else '' | |
| table = ' FROM items i LEFT JOIN human.notes h ON i.id=h.id' + where | |
| order = {'id':'i.id', 'duration_desc':'i.duration DESC,i.id', 'duration_asc':'i.duration,i.id', 'random':'random()'}.get(args.get('sort'), 'i.id') | |
| with closing(self.connect()) as db: | |
| count = db.execute('SELECT count(*)' + table, values).fetchone()[0] | |
| selected = db.execute('SELECT i.id,i.source,i.kind,i.status,i.split,i.question,i.answer,i.duration,i.voice_count,i.media,i.reason,h.verdict' + table + ' ORDER BY ' + order + ' LIMIT ? OFFSET ?', values + [limit, page*limit]) | |
| return dict(total=count, page=page, limit=limit, items=[dict(r) for r in selected]) | |
| def detail(self, item): | |
| with closing(self.connect()) as db: | |
| r = db.execute('SELECT * FROM items WHERE id=?', (item,)).fetchone() | |
| if r is None: raise KeyError(item) | |
| obj = dict(r); obj.update(json.loads(obj.pop('detail'))) | |
| h = db.execute('SELECT * FROM human.notes WHERE id=?', (item,)).fetchone() | |
| obj['human'] = dict(h) if h else {} | |
| for voice in obj['voices']: | |
| p = allowed_path(voice.get('asr_receipt'), self.data) | |
| if p and p.is_file() and p.stat().st_size < 256_000: | |
| voice['asr'] = read(p) | |
| return obj | |
| def annotate(self, obj): | |
| item, verdict, note = obj.get('id'), obj.get('verdict'), obj.get('note', '') | |
| if not isinstance(item, str) or not isinstance(verdict, str) or verdict not in {'pass', 'reject', 'unsure'} or not isinstance(note, str) or len(note) > 4000: | |
| raise ValueError('Invalid review') | |
| with closing(self.connect()) as db: | |
| if not db.execute('SELECT 1 FROM items WHERE id=?', (item,)).fetchone(): raise KeyError(item) | |
| db.execute('INSERT OR REPLACE INTO human.notes VALUES (?,?,?,?)', (item, verdict, note, time.time())) | |
| db.commit() | |
| def file(self, token): | |
| with closing(self.connect()) as db: | |
| r = db.execute('SELECT path FROM media WHERE id=?', (token,)).fetchone() | |
| p = allowed_path(r['path'], self.data) if r else None | |
| if not p or not p.is_file() or p.suffix.lower() not in {'.jpg', '.jpeg', '.png', '.webp', '.mp4', '.wav', '.mp3'}: | |
| raise KeyError(token) | |
| return p | |
| def handler(catalog): | |
| class Handler(BaseHTTPRequestHandler): | |
| def log_message(self, fmt, *args): | |
| if not args or str(args[1] if len(args)>1 else '') not in ('200', '206', '304'): | |
| super().log_message(fmt, *args) | |
| def send_headers(self, code, kind, length, extra=()): | |
| self.send_response(code) | |
| self.send_header('Content-Type', kind) | |
| self.send_header('Content-Length', str(length)) | |
| self.send_header('X-Content-Type-Options', 'nosniff') | |
| self.send_header('Referrer-Policy', 'no-referrer') | |
| self.send_header('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self'; media-src 'self'; connect-src 'self'; frame-ancestors 'none'") | |
| for k, v in extra: self.send_header(k, v) | |
| self.end_headers() | |
| def output(self, obj, code=200): | |
| b = json.dumps(obj, ensure_ascii=False).encode() | |
| self.send_headers(code, 'application/json; charset=utf-8', len(b), [('Cache-Control','no-store')]) | |
| if self.command != 'HEAD': self.wfile.write(b) | |
| def do_HEAD(self): self.do_GET() | |
| def do_GET(self): | |
| try: | |
| u = urlsplit(self.path); args = {k:v[0] for k,v in parse_qs(u.query).items()} | |
| if u.path in ('/', '/app.js', '/style.css', '/progress', '/progress.js'): | |
| p = HERE / {'/':'index.html','/app.js':'app.js','/style.css':'style.css', | |
| '/progress':'progress.html','/progress.js':'progress.js'}[u.path] | |
| b = p.read_bytes(); self.send_headers(200, mimetypes.guess_type(p)[0]+'; charset=utf-8', len(b)) | |
| if self.command != 'HEAD': self.wfile.write(b) | |
| elif u.path == '/api/audit-progress': | |
| status = read(catalog.state/'audit_progress.json', {}) | |
| status['stale'] = time.time()-status.get('updated_at', 0) > 180 | |
| self.output(status) | |
| elif u.path == '/api/stats': self.output(catalog.stats()) | |
| elif u.path == '/api/items': self.output(catalog.listing(args)) | |
| elif u.path == '/api/detail': self.output(catalog.detail(args['id'])) | |
| elif u.path == '/api/reviews': | |
| with closing(catalog.connect()) as db: | |
| self.output([dict(r) for r in db.execute('SELECT * FROM human.notes ORDER BY updated DESC')]) | |
| elif u.path.startswith('/media/'): | |
| p = catalog.file(u.path.removeprefix('/media/')) | |
| with p.open('rb') as f: | |
| size = os.fstat(f.fileno()).st_size | |
| try: start, end, partial = byte_range(self.headers.get('Range'), size) | |
| except ValueError: | |
| self.send_headers(416, 'text/plain', 0, [('Content-Range', f'bytes */{size}')]); return | |
| extra = [('Accept-Ranges','bytes'), ('Cache-Control','private, max-age=3600')] | |
| if partial: extra.append(('Content-Range',f'bytes {start}-{end}/{size}')) | |
| self.send_headers(206 if partial else 200, mimetypes.guess_type(p)[0] or 'application/octet-stream', end-start+1, extra) | |
| if self.command == 'HEAD': return | |
| f.seek(start); remaining = end-start+1 | |
| while remaining: | |
| block = f.read(min(256*1024, remaining)) | |
| if not block: break | |
| self.wfile.write(block); remaining -= len(block) | |
| else: self.output({'error':'Not found'}, 404) | |
| except (BrokenPipeError, ConnectionResetError): pass | |
| except KeyError: self.output({'error':'素材或条目不存在'}, 404) | |
| except (ValueError, TypeError): self.output({'error':'参数无效'}, 400) | |
| def do_POST(self): | |
| # Explicit browser same-origin check. Loopback service, no CORS. | |
| origin = self.headers.get('Origin', '') | |
| if (urlsplit(origin).scheme not in ('http', 'https') or | |
| urlsplit(origin).netloc != self.headers.get('Host') or | |
| self.headers.get('Content-Type', '').split(';')[0] != 'application/json'): | |
| self.output({'error':'Same-origin JSON required'}, 403); return | |
| try: | |
| if self.path != '/api/review': raise KeyError(self.path) | |
| size = int(self.headers.get('Content-Length', 0)) | |
| if not 0 < size <= 20000: raise ValueError('size') | |
| obj = json.loads(self.rfile.read(size)) | |
| if not isinstance(obj, dict): raise ValueError('body') | |
| catalog.annotate(obj); self.output({'saved': True}) | |
| except (ValueError, TypeError): self.output({'error':'无效审核内容'}, 400) | |
| except KeyError: self.output({'error':'条目不存在'}, 404) | |
| return Handler | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--data', type=Path, default=DEFAULT_DATA) | |
| p.add_argument('--state', type=Path, default=DEFAULT_STATE) | |
| p.add_argument('--port', type=int, default=8770) | |
| p.add_argument('--build', action='store_true') | |
| a = p.parse_args() | |
| if a.build: build(a.data.resolve(), a.state.resolve()); return | |
| c = Catalog(a.state.resolve(), a.data.resolve()) | |
| server = ThreadingHTTPServer(('127.0.0.1', a.port), handler(c)) | |
| print(f'VQA review dashboard: http://127.0.0.1:{a.port}/', flush=True) | |
| server.serve_forever() | |
| if __name__ == '__main__': main() | |