Download scripts/data/consolidate_visual_dataset.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 24.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/consolidate_visual_dataset.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/consolidate_visual_dataset.py
-
curl -L -o consolidate_visual_dataset.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/consolidate_visual_dataset.py
24.6 kB
| #!/usr/bin/env python3 | |
| """Non-destructive, resumable CPU-only visual dataset consolidation. | |
| Hardlinks fail closed across filesystems (never silently copy a TB). Read-only | |
| source manifests and immutable audit receipts remain usable by live workers. | |
| Only the two destination directories images/ and videos/ are created. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter, defaultdict, deque | |
| from concurrent.futures import ThreadPoolExecutor | |
| from datetime import datetime, timezone | |
| import fcntl | |
| from functools import lru_cache | |
| import errno | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import statistics | |
| import sys | |
| import time | |
| import unicodedata | |
| import wave | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) | |
| from psi.data.seed.visual_dataset import FORMAT, VOICES | |
| from psi.data.seed.causal_video_qa import valid_point | |
| def retry_io(action, context): | |
| for attempt in range(4): | |
| try: | |
| return action() | |
| except OSError as exc: | |
| if exc.errno not in (errno.EIO, errno.ESTALE, errno.ETIMEDOUT) or attempt == 3: | |
| raise OSError(exc.errno, f"{context}: {exc}") from exc | |
| print(json.dumps(dict(stage="transient_io_retry", context=str(context), attempt=attempt+1)), flush=True) | |
| time.sleep(.25 * (2 ** attempt)) | |
| def _digest(path): | |
| h = hashlib.sha256() | |
| with Path(path).open("rb") as handle: | |
| for block in iter(lambda: handle.read(4 << 20), b""): | |
| h.update(block) | |
| return h.hexdigest() | |
| def digest(path): | |
| return retry_io(lambda: _digest(path), path) | |
| def bounded_map(pool, fn, items, workers): | |
| """Bound in-flight metadata work and cancel queued work after a failure.""" | |
| source = iter(items) | |
| pending = deque() | |
| for _ in range(workers * 2): | |
| item = next(source, None) | |
| if item is not None: | |
| pending.append(pool.submit(fn, item)) | |
| try: | |
| while pending: | |
| yield pending.popleft().result() | |
| item = next(source, None) | |
| if item is not None: | |
| pending.append(pool.submit(fn, item)) | |
| finally: | |
| for future in pending: | |
| future.cancel() | |
| def atomic_json(path, data): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_suffix(path.suffix + ".partial") | |
| with tmp.open("w") as handle: | |
| json.dump(data, handle, ensure_ascii=False, separators=(",", ":")) | |
| handle.flush() | |
| os.fsync(handle.fileno()) | |
| os.replace(tmp, path) | |
| def norm(text): | |
| return " ".join(unicodedata.normalize("NFKC", text).casefold().split()) | |
| def qa_identity(media_id, row): | |
| # Never collapse different answers or genuinely different questions. | |
| return hashlib.sha256(json.dumps([media_id, norm(row["question"]), norm(row["answer"])], | |
| ensure_ascii=False).encode()).hexdigest() | |
| def resolved_parent(parent): | |
| return Path(parent).resolve(strict=True) | |
| def ensure_directory(path): | |
| Path(path).mkdir(parents=True, exist_ok=True) | |
| def source_path(source): | |
| source = Path(source) | |
| source = resolved_parent(str(source.parent)) / source.name | |
| if source.is_symlink(): | |
| source = source.resolve(strict=True) | |
| return source | |
| def link_file(source, folder, relative): | |
| source = source_path(source) | |
| target = folder / relative | |
| ensure_directory(str(target.parent)) | |
| try: | |
| os.link(source, target) | |
| except FileExistsError: | |
| if not os.path.samefile(source, target): | |
| # CAS receipts can have equal bytes but separate inodes. | |
| if source.stat().st_size != target.stat().st_size or digest(source) != digest(target): | |
| raise ValueError(f"Conflicting existing file: {target}") | |
| return str(relative) | |
| def receipt_link(source, folder): | |
| sha = digest(source) | |
| return link_file(source, folder, Path("receipts") / sha[:2] / (sha + ".json")) | |
| def media_check(item): | |
| path, expected, kind = item | |
| actual = digest(path) | |
| if actual != expected: | |
| raise ValueError(f"Media SHA mismatch: {path}") | |
| if kind == "images": | |
| from PIL import Image | |
| with Image.open(path) as image: | |
| image = image.convert("RGB") | |
| content = hashlib.sha256(str(image.size).encode() + image.tobytes()).hexdigest() | |
| else: | |
| content = actual | |
| return path, content | |
| def durations(values): | |
| edges = [(0, 10), (10, 20), (20, 30), (30, 60), (60, 120), (120, float("inf"))] | |
| return dict(count=len(values), min=min(values) if values else None, | |
| median=statistics.median(values) if values else None, | |
| mean=statistics.mean(values) if values else None, | |
| max=max(values) if values else None, | |
| bins={f"{lo}-{hi}s": sum(lo <= v < hi for v in values) for lo, hi in edges}) | |
| def load_groups(manifests): | |
| groups = [] | |
| inputs = [] | |
| for manifest in manifests: | |
| raw = Path(manifest).read_bytes() | |
| sha = hashlib.sha256(raw).hexdigest() | |
| obj = json.loads(raw) | |
| if obj.get("format") != "hfd_overlay_assets_v1": | |
| raise ValueError(f"Unknown manifest: {manifest}") | |
| release = Path(manifest).parent / "admission/acceptance.json" | |
| released = False | |
| if release.exists(): | |
| mark = json.loads(release.read_text()) | |
| released = mark.get("passed") is True and mark.get("manifest_sha256") == sha | |
| local = defaultdict(list) | |
| for row in obj["assets"]: | |
| local[row["asset_id"]].append(row) | |
| for rows in local.values(): | |
| groups.append(dict(rows=rows, source_manifest=str(Path(manifest).resolve()), | |
| source_sha256=sha, released=released)) | |
| inputs.append(dict(path=str(Path(manifest).resolve()), sha256=sha, | |
| groups=len(local), rows=len(obj["assets"]), | |
| provenance=obj.get("provenance", {}))) | |
| print(json.dumps(dict(stage="read_manifest", groups=len(local), rows=len(obj["assets"]))), flush=True) | |
| return groups, inputs | |
| def audit_snapshot(root): | |
| result = {} | |
| if root is None: | |
| return result | |
| # No partial files, no moving target manifest; each review is immutable. | |
| for shard in sorted(Path(root).glob("shard*")): | |
| contract_path = shard / "contract.json" | |
| contract = json.loads(contract_path.read_text()) | |
| if contract.get("version") != "causal_prefix_qa_v2": | |
| raise ValueError("Incompatible causal review policy") | |
| sha = digest(contract_path) | |
| for path in sorted((shard / "reviews").glob("*.json")): | |
| r = json.loads(path.read_text()) | |
| if r["contract_sha256"] != sha or r["asset_id"] in result: | |
| raise ValueError(f"Invalid/duplicate audit: {path}") | |
| result[r["asset_id"]] = (path, r, contract["source_sha256"]) | |
| return result | |
| def admission(group, audits): | |
| row = group["rows"][0] | |
| if not group["released"]: | |
| return "pending", {} | |
| if not row.get("video_path"): | |
| return "accepted", {} | |
| entry = audits.get(row["asset_id"]) | |
| if entry: | |
| path, review, source_sha = entry | |
| if review["video_sha256"] != row["video_sha256"] or source_sha != group["source_sha256"]: | |
| raise ValueError(f"Audit/source identity mismatch: {row['asset_id']}") | |
| points = review.get("causal_qa_points", []) | |
| if review.get("accepted") is True: | |
| if not points or not all(valid_point(p, row["duration_seconds"]) for p in points): | |
| raise ValueError("Invalid causal QA times") | |
| return "accepted", dict(causal_qa_points=points, live_qa_receipt=str(path), | |
| earliest_answer_video_seconds=min(p["question_end_seconds"] for p in points)) | |
| return "rejected", dict(live_qa_receipt=str(path)) | |
| if group["released"] and row.get("live_qa_receipt") and ( | |
| row.get("live_qa_windows") or row.get("causal_qa_points")): | |
| return "accepted", {k: row[k] for k in ("live_qa_receipt", "live_qa_windows", | |
| "causal_qa_points", "earliest_answer_video_seconds") if k in row} | |
| return "pending", {} | |
| def prepare_group(group, identity, audits): | |
| rows = group["rows"] | |
| by_voice = {r["voice"]: r for r in rows} | |
| if len(rows) != 8 or set(by_voice) != set(VOICES): | |
| raise ValueError(f"Not exactly eight distinct voices: {rows[0]['asset_id']}") | |
| row = by_voice["female_01_en"] | |
| kind = "videos" if row.get("video_path") else "images" | |
| media_key = "video" if kind == "videos" else "image" | |
| for r in rows: | |
| if (r.get(media_key + "_sha256"), r["split"], r.get("visual_features_path")) != ( | |
| row[media_key + "_sha256"], row["split"], row.get("visual_features_path")): | |
| raise ValueError(f"Voice variants disagree: {row['asset_id']}") | |
| for lang in ("en", "zh"): | |
| texts = {(norm(r["question"]), norm(r["answer"])) for r in rows if r["voice"].endswith(lang)} | |
| if len(texts) != 1: | |
| raise ValueError(f"Voices disagree on QA text: {row['asset_id']}") | |
| if not row["question"].strip() or not row["answer"].strip(): | |
| raise ValueError("Empty QA") | |
| if kind == "videos" and not (0 < float(row["duration_seconds"]) < 86400): | |
| raise ValueError("Invalid video duration") | |
| status, fields = admission(group, audits) | |
| return dict(group=group, row=row, by_voice=by_voice, kind=kind, media_key=media_key, | |
| media_id=identity, qa_id=qa_identity(identity, row), status=status, fields=fields) | |
| def build_record(items, destination): | |
| first = items[0] | |
| row, kind, media_id = first["row"], first["kind"], first["media_id"] | |
| folder = destination / kind | |
| media = row[first["media_key"] + "_path"] | |
| # Features are tied to the selected exact media bytes, never to a near duplicate. | |
| feature = row["visual_features_path"] | |
| if Path(feature).stat().st_size == 0: | |
| raise ValueError(f"Empty feature: {feature}") | |
| record = dict(id=media_id, kind=kind, split=row["split"], | |
| media_path=link_file(media, folder, Path("media") / media_id[:2] / (media_id + Path(media).suffix.lower())), | |
| media_sha256=row[first["media_key"] + "_sha256"], | |
| features_path=link_file(feature, folder, Path("features") / media_id[:2] / (media_id + ".pt")), | |
| duration_seconds=float(row["duration_seconds"]) if kind == "videos" else None, | |
| questions=[]) | |
| if row.get("parent_video_id"): | |
| record["parent_video_id"] = row["parent_video_id"] | |
| audio_bytes = 0 | |
| for item in items: | |
| qa = dict(id=item["qa_id"], status=item["status"], voices={}, **item["fields"]) | |
| qa["provenance"] = dict(asset_id=item["row"]["asset_id"], | |
| manifest_sha256=item["group"]["source_sha256"]) | |
| for key in ("visual_receipt", "live_qa_receipt"): | |
| source = qa.get(key) or item["row"].get(key) | |
| if source: | |
| qa[key] = receipt_link(source, folder) | |
| for voice in VOICES: | |
| r = item["by_voice"][voice] | |
| source = r["question_audio_path"] | |
| actual = digest(source) | |
| if actual != r["question_audio_sha256"]: | |
| raise ValueError(f"Audio SHA mismatch: {source}") | |
| with wave.open(source, "rb") as wav: | |
| if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2) or wav.getnframes() == 0: | |
| raise ValueError(f"Invalid question audio: {source}") | |
| seconds = wav.getnframes() / wav.getframerate() | |
| audio_bytes += Path(source).stat().st_size | |
| values = dict(question=r["question"], answer=r["answer"], | |
| audio_path=link_file(source, folder, Path("seed_audio") / qa["id"] / (voice + ".wav")), | |
| audio_sha256=actual, audio_seconds=seconds) | |
| for key in ("asr_receipt", "question_translation_receipt", "answer_translation_receipt"): | |
| if r.get(key): | |
| values[key] = receipt_link(r[key], folder) | |
| qa["voices"][voice] = values | |
| record["questions"].append(qa) | |
| return record, audio_bytes | |
| def consolidate(manifests, output, *, audit_root=None, workers=8): | |
| output = Path(output).resolve() | |
| for kind in ("images", "videos"): | |
| (output / kind).mkdir(parents=True, exist_ok=True) | |
| with (output / "images/build.lock").open("a") as lock: | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| return _build(manifests, output, audit_root, workers) | |
| def _build(manifests, output, audit_root, workers): | |
| started = time.time() | |
| generation = datetime.now(timezone.utc).isoformat() | |
| groups, inputs = load_groups(manifests) | |
| audits = audit_snapshot(audit_root) | |
| print(json.dumps(dict(stage="audit_snapshot", receipts=len(audits))), flush=True) | |
| media_inputs = {} | |
| for g in groups: | |
| r = g["rows"][0] | |
| key = "video" if r.get("video_path") else "image" | |
| path = str(source_path(r[key + "_path"])) | |
| media_inputs[path] = (path, r[key + "_sha256"], key + "s") | |
| identities = {} | |
| cache_path = output / "images/media_checks.json" | |
| cache = json.loads(cache_path.read_text()) if cache_path.exists() else {} | |
| signatures = {} | |
| pending_media = [] | |
| for path, item in media_inputs.items(): | |
| stat = Path(path).stat() | |
| signature = [item[1], stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns] | |
| signatures[path] = signature | |
| if cache.get(path, {}).get("signature") == signature: | |
| identities[path] = cache[path]["identity"] | |
| else: | |
| pending_media.append(item) | |
| last = time.time() | |
| with ThreadPoolExecutor(max_workers=workers) as pool: | |
| for path, identity in bounded_map(pool, lambda x: retry_io(lambda: media_check(x), x[0]), pending_media, workers): | |
| identities[path] = identity | |
| if time.time() - last > 30: | |
| print(json.dumps(dict(stage="verify_media", done=len(identities), total=len(media_inputs))), flush=True) | |
| last = time.time() | |
| print(json.dumps(dict(stage="verify_media_complete", total=len(identities))), flush=True) | |
| atomic_json(cache_path, {p: dict(signature=signatures[p], identity=i) for p, i in identities.items()}) | |
| unique = {} | |
| exclusions = [] | |
| duplicate_qa = 0 | |
| split_ids = defaultdict(set) | |
| for group in groups: | |
| row = group["rows"][0] | |
| path = str(source_path(row.get("video_path") or row["image_path"])) | |
| item = prepare_group(group, identities[path], audits) | |
| split_ids[item["media_id"]].add(row["split"]) | |
| if row.get("parent_video_id"): | |
| split_ids["parent:" + row["parent_video_id"]].add(row["split"]) | |
| key = item["qa_id"] | |
| if key in unique: | |
| duplicate_qa += 1 | |
| old = unique[key] | |
| # Preserve the strongest audit, deterministic tie-break; do not mix voices. | |
| priority = {"accepted": 2, "pending": 1, "rejected": 0} | |
| if priority[item["status"]] > priority[old["status"]]: | |
| unique[key] = item | |
| else: | |
| unique[key] = item | |
| grouped = defaultdict(list) | |
| for item in unique.values(): | |
| r = item["row"] | |
| conflicts = len(split_ids[item["media_id"]]) > 1 or ( | |
| r.get("parent_video_id") and len(split_ids["parent:" + r["parent_video_id"]]) > 1) | |
| if conflicts or item["status"] == "rejected": | |
| exclusions.append(dict(kind=item["kind"], id=item["qa_id"], asset_id=r["asset_id"], | |
| reason="split_conflict" if conflicts else "review_rejected")) | |
| continue | |
| grouped[item["media_id"]].append(item) | |
| # Protect reruns: selected media/feature provider remains deterministic. | |
| jobs = [sorted(v, key=lambda x: x["qa_id"]) for _, v in sorted(grouped.items())] | |
| counts = {k: Counter() for k in ("images", "videos")} | |
| length = {k: [] for k in ("all", "accepted")} | |
| handles = {k: (output / k / "manifest.jsonl.partial").open("w") for k in counts} | |
| last = time.time() | |
| try: | |
| with ThreadPoolExecutor(max_workers=workers) as pool: | |
| work = lambda x: retry_io(lambda: build_record(x, output), x[0]["qa_id"]) | |
| for i, (record, audio_bytes) in enumerate(bounded_map(pool, work, jobs, workers), 1): | |
| kind = record["kind"] | |
| handles[kind].write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") | |
| c = counts[kind] | |
| c["media"] += 1 | |
| c["questions"] += len(record["questions"]) | |
| c["question_audio_bytes"] += audio_bytes | |
| c["voice_rows"] += 8 * len(record["questions"]) | |
| statuses = {q["status"] for q in record["questions"]} | |
| for q in record["questions"]: | |
| c[q["status"] + "_questions"] += 1 | |
| if "accepted" in statuses: | |
| c["accepted_media"] += 1 | |
| if kind == "videos": | |
| length["all"].append(record["duration_seconds"]) | |
| if "accepted" in statuses: | |
| length["accepted"].append(record["duration_seconds"]) | |
| if time.time() - last > 30: | |
| progress = dict(stage="verify_and_link", done=i, total=len(jobs), elapsed=time.time()-started) | |
| atomic_json(output / "images/build_progress.json", progress) | |
| print(json.dumps(progress), flush=True) | |
| last = time.time() | |
| for handle in handles.values(): | |
| handle.flush() | |
| os.fsync(handle.fileno()) | |
| finally: | |
| for handle in handles.values(): | |
| handle.close() | |
| result = {} | |
| for kind in counts: | |
| folder = output / kind | |
| os.replace(folder / "manifest.jsonl.partial", folder / "manifest.jsonl") | |
| atomic_json(folder / "inputs.json", inputs) | |
| atomic_json(folder / "excluded.json", [e for e in exclusions if e["kind"] == kind]) | |
| summary = dict(format=FORMAT, generation=generation, counts=dict(counts[kind]), | |
| expected_voices=list(VOICES), all_questions_have_eight_voices=True, | |
| manifest_sha256=digest(folder / "manifest.jsonl"), | |
| source_groups=len(groups), duplicate_qa_removed=duplicate_qa, | |
| audit_receipts_snapshot=len(audits), | |
| dedup="image decoded RGB identity; video exact file SHA256; QA normalized English question+answer per visual; train/val parent conflicts excluded", | |
| not_claimed="No semantic/perceptual video near-duplicate or human quality guarantee", | |
| storage="hardlinks; no source deletion; immutable files must not be edited in place") | |
| if kind == "videos": | |
| summary["duration_seconds"] = {k: durations(v) for k, v in length.items()} | |
| atomic_json(folder / "summary.json", summary) | |
| result[kind] = summary | |
| atomic_json(output / "images/build_progress.json", dict(stage="complete", elapsed=time.time()-started)) | |
| return result | |
| def refresh_reviews(output, audit_root): | |
| """Incrementally admit existing pending records, without touching media/voices. | |
| Metadata and copied immutable receipts only. A failed refresh leaves the old | |
| publication intact or causes the reader to fail closed on generation mismatch. | |
| """ | |
| output = Path(output).resolve() | |
| with (output / "images/build.lock").open("a") as lock: | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| summaries = {k: json.loads((output/k/"summary.json").read_text()) for k in ("images", "videos")} | |
| if summaries["images"]["generation"] != summaries["videos"]["generation"]: | |
| raise ValueError("Incomplete publication; rebuild before refreshing") | |
| audits = audit_snapshot(audit_root) | |
| changed = 0 | |
| counts = Counter() | |
| length = {"all": [], "accepted": []} | |
| folder = output / "videos" | |
| tmp = folder / "manifest.jsonl.partial" | |
| with (folder / "manifest.jsonl").open() as source, tmp.open("w") as dest: | |
| for line in source: | |
| row = json.loads(line) | |
| for qa in row["questions"]: | |
| if qa["status"] != "pending": | |
| continue | |
| entry = audits.get(qa["provenance"]["asset_id"]) | |
| if entry is None: | |
| continue | |
| path, review, source_sha = entry | |
| if review["video_sha256"] != row["media_sha256"] or source_sha != qa["provenance"]["manifest_sha256"]: | |
| raise ValueError("Review source changed") | |
| points = review.get("causal_qa_points", []) | |
| if review["accepted"] and (not points or not all(valid_point(p, row["duration_seconds"]) for p in points)): | |
| raise ValueError("Invalid new causal points") | |
| qa["status"] = "accepted" if review["accepted"] else "rejected" | |
| qa["live_qa_receipt"] = receipt_link(path, folder) | |
| if review["accepted"]: | |
| qa["causal_qa_points"] = points | |
| qa["earliest_answer_video_seconds"] = min(p["question_end_seconds"] for p in points) | |
| changed += 1 | |
| counts["media"] += 1 | |
| counts["questions"] += len(row["questions"]) | |
| counts["voice_rows"] += 8 * len(row["questions"]) | |
| for qa in row["questions"]: | |
| counts[qa["status"] + "_questions"] += 1 | |
| length["all"].append(row["duration_seconds"]) | |
| if any(q["status"] == "accepted" for q in row["questions"]): | |
| counts["accepted_media"] += 1 | |
| length["accepted"].append(row["duration_seconds"]) | |
| dest.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n") | |
| dest.flush() | |
| os.fsync(dest.fileno()) | |
| if not changed: | |
| tmp.unlink() | |
| return dict(changed=0) | |
| # First invalidate generations, so no reader can see a mixed release. | |
| generation = datetime.now(timezone.utc).isoformat() | |
| summaries["images"]["generation"] = generation | |
| atomic_json(output / "images/summary.json", summaries["images"]) | |
| os.replace(tmp, folder / "manifest.jsonl") | |
| video = summaries["videos"] | |
| video["generation"] = generation | |
| video["counts"].update(counts) | |
| video["counts"]["pending_questions"] = counts["pending_questions"] | |
| video["duration_seconds"] = {k: durations(v) for k, v in length.items()} | |
| video["manifest_sha256"] = digest(folder / "manifest.jsonl") | |
| video["audit_receipts_snapshot"] = len(audits) | |
| atomic_json(folder / "summary.json", video) | |
| return dict(changed=changed, counts=video["counts"]) | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument("--manifests", type=Path, nargs="+") | |
| p.add_argument("--output", type=Path, required=True) | |
| p.add_argument("--audit-root", type=Path) | |
| p.add_argument("--workers", type=int, default=8) | |
| p.add_argument("--refresh-reviews", action="store_true", help="Only update review status of packaged videos") | |
| p.add_argument("--watch", action="store_true", help="Refresh every 30 min until all packaged videos are reviewed") | |
| a = p.parse_args() | |
| if a.workers < 1: | |
| p.error("workers must be positive") | |
| if a.refresh_reviews: | |
| if not a.audit_root: | |
| p.error("--refresh-reviews requires --audit-root") | |
| while True: | |
| print(json.dumps(refresh_reviews(a.output, a.audit_root), ensure_ascii=False), flush=True) | |
| summary = json.loads((a.output / "videos/summary.json").read_text()) | |
| if not a.watch or summary["counts"].get("pending_questions", 0) == 0: | |
| break | |
| time.sleep(1800) | |
| else: | |
| if not a.manifests: | |
| p.error("--manifests is required for building") | |
| print(json.dumps(consolidate(a.manifests, a.output, audit_root=a.audit_root, workers=a.workers), ensure_ascii=False)) | |
| if __name__ == "__main__": | |
| main() | |