code-HFD / scripts /data /encode_swa_vqav2.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
23.3 kB
#!/usr/bin/env python3
"""Offline VQAv2 -> spoken, causal SWA sources with independent admission gates.
Stages may overlap: voices -> asr, independent image review, then encode when
both reviews exist. GPU phases accept rank/world sharding; no distributed model
or shared KV is used. All phases are resumable. Frozen inputs are factorized instead of copying the same image and
base-motion audio into dozens of windows on disk. Qwen hidden/KV are NOT cached.
"""
from __future__ import annotations
import argparse
from collections import Counter
from functools import lru_cache
import hashlib
import json
import math
from pathlib import Path
import re
import sys
import time
import numpy as np
from PIL import Image, ImageOps
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src"))
from psi.data.seed.qwen_streaming_audio import prepare_qwen_streaming_audio_window
from prepare_swa_vqav2 import save_json, normalized, conservative_question
from swa_pipeline_queue import ready_batches
SYSTEM = ("You are Qwen, a robot with a live camera and microphone. Camera frames and audio arrive in time order. "
"Execute motion commands. Answer visual questions briefly using the current camera view while continuing the requested motion. "
"A visual question is not a new motion command. Do not answer before the question is complete.")
def sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(2**20), b""):
digest.update(block)
return digest.hexdigest()
def atomic_torch(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(".partial.pt")
torch.save(value, temp)
temp.replace(path)
def read(path):
return json.loads(Path(path).read_text())
def canonical_answer(text):
words = normalized(text).split()
numbers = dict(zip("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty".split(), map(str, range(21))))
return " ".join(numbers.get(w, w) for w in words if w not in {"a", "an", "the"})
def canonical_speech(text):
# Punctuation/case and integer spellings only; never ignore missing words.
numbers = dict(zip("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty".split(), map(str, range(21))))
return " ".join(numbers.get(w, w) for w in normalized(text).split())
def voice_jobs(args):
bodies = read(args.body_plan)
reference = next(b for b in bodies if b["split"] == "train" and b["voice"] == "female_01_en" and len(b["instruction"].split()) >= 4)
refsha = sha(reference["reference_audio"])
jobs = {}
for row in read(args.root / "candidates.json"):
key = hashlib.sha256((refsha + "\0" + row["question"]).encode()).hexdigest()[:32]
jobs[row["question"]] = dict(key=key, text=row["question"], reference_audio=reference["reference_audio"],
reference_text=reference["instruction"], reference_sha256=refsha)
return jobs
def shard(rows, args):
return rows[args.rank::args.world][:args.limit or None]
def work_batches(args, rows, output_path, input_paths):
return ready_batches(rows, output_path=output_path, input_paths=input_paths,
batch_size=args.batch, follow=args.follow, upstream_done=args.upstream_done,
poll_seconds=args.poll_seconds, timeout_seconds=args.upstream_timeout)
def voices(args):
sys.path.insert(0, str(args.tts_deps))
import soundfile as sf
from scipy.signal import resample_poly
from qwen_tts import Qwen3TTSModel
jobs = sorted(voice_jobs(args).values(), key=lambda r: r["key"])
pending = [r for r in shard(jobs, args) if not (args.root / "voices" / (r["key"] + ".json")).is_file()]
if not pending:
return
model = Qwen3TTSModel.from_pretrained(str(args.tts_model), device_map="cuda:0", dtype=torch.bfloat16, attn_implementation="sdpa")
ref = pending[0]
prompt = model.create_voice_clone_prompt(ref_audio=ref["reference_audio"], ref_text=ref["reference_text"], x_vector_only_mode=False)[0]
pending.sort(key=lambda r: (len(r["text"].split()), r["key"]))
start = time.monotonic()
for offset in range(0, len(pending), args.batch):
batch = pending[offset:offset + args.batch]
waves, sr = model.generate_voice_clone(text=[r["text"] for r in batch], language=["English"] * len(batch),
voice_clone_prompt=[prompt] * len(batch), do_sample=False, subtalker_dosample=False, max_new_tokens=300)
for row, wave in zip(batch, waves):
g = math.gcd(int(sr), 16000)
wave = resample_poly(np.asarray(wave, np.float32), 16000 // g, int(sr) // g).astype(np.float32)
path = args.root / "voices" / (row["key"] + ".wav");path.parent.mkdir(parents=True, exist_ok=True)
if not np.isfinite(wave).all() or not .3 <= len(wave) / 16000 <= 15:
save_json(path.with_suffix(".json"), {**row, "rejected": "waveform_quality"});continue
sf.write(path.with_suffix(".partial.wav"), wave, 16000, subtype="PCM_16")
path.with_suffix(".partial.wav").replace(path)
save_json(path.with_suffix(".json"), {**row, "seconds": len(wave) / 16000, "wave_sha256": sha(path), "voice": "female_01_en"})
print("TTS", args.rank, offset + len(batch), len(pending), "items/s", round((offset + len(batch)) / (time.monotonic() - start), 2), flush=True)
def asr(args):
import soundfile as sf
from transformers import WhisperForConditionalGeneration, WhisperProcessor
jobs = sorted(voice_jobs(args).values(), key=lambda r: r["key"])
processor = WhisperProcessor.from_pretrained(str(args.asr_model), local_files_only=True)
model = WhisperForConditionalGeneration.from_pretrained(str(args.asr_model), dtype=torch.float16, local_files_only=True).cuda().eval()
model.generation_config.forced_decoder_ids = None
rows = shard(jobs, args);processed = 0;started = time.monotonic()
batches = work_batches(args, rows,
lambda r: args.root / "asr" / (r["key"] + ".json"),
lambda r: [args.root / "voices" / (r["key"] + ".json")])
for batch in batches:
active, waves = [], []
for row in batch:
path = args.root / "voices" / (row["key"] + ".wav")
receipt = read(path.with_suffix(".json"))
if "rejected" in receipt:
save_json(args.root / "asr" / (row["key"] + ".json"), {"accepted": False, "reason": receipt["rejected"]});continue
assert sha(path) == receipt["wave_sha256"]
wave, sr = sf.read(path, dtype="float32");assert sr == 16000
active.append((row, receipt));waves.append(wave)
if waves:
inputs = processor(waves, sampling_rate=16000, return_tensors="pt", return_attention_mask=True)
inputs = {k: v.cuda().to(dtype=torch.float16 if v.is_floating_point() else v.dtype) for k, v in inputs.items()}
with torch.inference_mode():
outputs = model.generate(**inputs, language="en", task="transcribe", max_new_tokens=100)
texts = processor.batch_decode(outputs, skip_special_tokens=True)
for (row, receipt), heard in zip(active, texts):
save_json(args.root / "asr" / (row["key"] + ".json"), {
"accepted": canonical_speech(heard) == canonical_speech(row["text"]),
"expected": row["text"], "heard": heard, "model": str(args.asr_model),
"wave_sha256": receipt["wave_sha256"], "independent_asr": True})
processed += len(batch)
print("ASR", args.rank, processed, len(rows), "items/s", round(processed / (time.monotonic() - started), 2), flush=True)
def review(args):
from transformers import AutoModelForImageTextToText, AutoProcessor
rows = shard(read(args.root / "image_checked.json"), args)
pending = [r for r in rows if not (args.root / "visual_review" / (r["id"] + ".json")).exists()]
if not pending:
return
processor = AutoProcessor.from_pretrained(str(args.review_model), local_files_only=True)
processor.tokenizer.padding_side = "left"
model = AutoModelForImageTextToText.from_pretrained(str(args.review_model), dtype=torch.bfloat16,
attn_implementation="flash_attention_2", local_files_only=True).cuda().eval()
start = time.monotonic()
for offset in range(0, len(pending), args.batch):
batch = pending[offset:offset + args.batch];images = [];texts = []
for row in batch:
with Image.open(row["image_path"]) as image:
images.append(ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0)))
conversation = [{"role": "user", "content": [{"type": "image"},
{"type": "text", "text": row["question"] + " Answer with only a short phrase, number, or yes/no."}]}]
texts.append(processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True))
inputs = processor(text=texts, images=images, padding=True, return_tensors="pt").to("cuda")
with torch.inference_mode():
generated = model.generate(**inputs, max_new_tokens=16, do_sample=False)
decoded = processor.batch_decode(generated[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)
for row, prediction in zip(batch, decoded):
save_json(args.root / "visual_review" / (row["id"] + ".json"), {
"accepted": canonical_answer(prediction) == canonical_answer(row["answer"]),
"prediction": prediction, "reference": row["answer"], "image_sha256": row["image_sha256"],
"model": str(args.review_model), "blind_to_reference": True, "camera_side": 224})
print("REVIEW", args.rank, offset + len(batch), len(pending), "items/s", round((offset + len(batch)) / (time.monotonic() - start), 2), flush=True)
@lru_cache(maxsize=32)
def body(path):
return torch.load(path, weights_only=False, mmap=True)
def camera_groups(count):
available = .6 + .4 * np.arange(count)
times = np.arange(math.floor((available[-1] + 1e-7) * 8) + 1) / 8
ends = np.searchsorted(times, available + 1e-7, side="right")
groups = [list(range(int(start), int(end))) for start, end in zip(np.r_[0, ends[:-1]], ends)]
return available, groups
def video_inputs(processor, image, count):
frames = np.repeat(np.asarray(image)[None], count, axis=0)
return processor.video_processor(videos=[torch.from_numpy(frames.copy()).permute(0, 3, 1, 2)],
do_sample_frames=False, size={"shortest_edge": 224**2, "longest_edge": 224**2}, return_tensors="pt")
@torch.inference_mode()
def make_base(model, processor, row, root):
path = root / "bases" / (row["id"] + ".pt")
if path.exists():
return path
src = body(row["path"]);steps = [s for s in src["source"]["steps"] if s["update_semantic_cache"]]
available, groups = camera_groups(len(steps));conf = model.config
prefix = processor.tokenizer.encode('<|im_start|>system\n' + SYSTEM + '<|im_end|>\n<|im_start|>user\n', add_special_tokens=False)
packets = []
for i, step in enumerate(steps):
feat = step["stream_input_features"][None].cuda();mask = step["stream_feature_attention_mask"][None].cuda()
audio = model.get_audio_features(feat, mask)
grid = torch.tensor([[math.ceil(len(groups[i]) / 2), 16, 16]], device="cuda")
nvideo = int(grid.prod()) // 4
ids = torch.tensor([(prefix if i == 0 else []) + [conf.vision_start_token_id] + [conf.video_token_id] * nvideo
+ [conf.vision_end_token_id, conf.audio_start_token_id] + [conf.audio_token_id] * len(audio) + [conf.audio_end_token_id]], device="cuda")
pos, _ = model.get_rope_index(ids, video_grid_thw=grid, second_per_grids=torch.tensor([.25], device="cuda"),
attention_mask=torch.ones_like(ids), audio_seqlens=mask.sum(-1))
emb = model.get_input_embeddings()(ids)[0];amask = ids[0].eq(conf.audio_token_id);vmask = ids[0].eq(conf.video_token_id)
emb[amask] = audio.to(emb);emb[vmask] = 0
packets.append(dict(ids=ids[0].cpu(), embeds=emb.cpu(), ordinary_audio=audio.cpu(), audio_mask=amask.cpu(), visual_mask=vmask.cpu(),
positions=pos[:, 0].cpu(), next_position=int(pos.max()) + 1, reply_label=0, answer_target="", available_seconds=float(available[i]),
video_pts=[j / 8 for j in groups[i]], video_grid=grid[0].cpu(), second_per_grid=.25, camera_indices=groups[i]))
atomic_torch(path, {"packets": packets, "system": SYSTEM, "body_path": row["path"], "body_sha256": sha(row["path"]), "base_model": str(model.name_or_path)})
return path
def encode(args):
import soundfile as sf
from transformers import Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration
rows = shard(read(args.root / "image_checked.json"), args);jobs = voice_jobs(args)
model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(str(args.base_model), dtype=torch.bfloat16,
attn_implementation="flash_attention_2", local_files_only=True).cuda().eval().requires_grad_(False)
processor = Qwen2_5OmniProcessor.from_pretrained(str(args.base_model), local_files_only=True)
bodies = [r for r in read(args.body_plan) if r["voice"] == "female_01_en"]
# Per-rank base copies avoid concurrent writes; finalization deduplicates
# them only after all workers complete. Their total size is small.
rankroot = args.root / f"components_r{args.rank}"
batches = work_batches(args, rows,
lambda r: args.root / "admitted" / (r["id"] + ".json"),
lambda r: ([args.root / "asr" / (jobs[r["question"]]["key"] + ".json"),
args.root / "visual_review" / (r["id"] + ".json")]
if conservative_question(r["question"]) else []))
for number, row in enumerate(row for batch in batches for row in batch):
output = args.root / "admitted" / (row["id"] + ".json")
if not conservative_question(row["question"]):
save_json(output, {"id": row["id"], "rejected": "not_conservative_visible_state"})
continue
if output.exists():
continue
job = jobs[row["question"]];audio_review = read(args.root / "asr" / (job["key"] + ".json"))
visual_review = read(args.root / "visual_review" / (row["id"] + ".json"))
if not audio_review["accepted"] or not visual_review["accepted"]:
save_json(output, {"id": row["id"], "rejected": "independent_asr_or_visual_gate"});continue
wav = args.root / "voices" / (job["key"] + ".wav")
if sha(wav) != audio_review["wave_sha256"] or sha(row["image_path"]) != visual_review["image_sha256"]:
raise RuntimeError("Reviewed input content changed")
question, sr = sf.read(wav, dtype="float32");assert sr == 16000
candidates = sorted((b for b in bodies if b["split"] == row["split"]),
key=lambda b: hashlib.sha256((row["id"] + b["id"]).encode()).hexdigest())
chosen = None
for b in candidates:
src = body(b["path"]);steps = [s for s in src["source"]["steps"] if s["update_semantic_cache"]]
motion_start = next((s["step_index"] / 50 for s in steps if s["control_label"] == 1), None)
if motion_start is None:
continue
qend = .6 + .4 * math.ceil((motion_start + 1.2 + len(question) / sr - .6) / .4)
qstart = qend - len(question) / sr;qw = int(round((qend - .6) / .4))
if qend + 2.0 <= src["source_seconds"] and qw + 3 < len(steps) and all(s["control_label"] == 1 for s in steps[max(0, int(qstart / .4)):qw + 3]):
chosen = b, src, steps, qstart, qend, qw;break
if chosen is None:
save_json(output, {"id": row["id"], "rejected": "no_unchanged_body_fits"});continue
b, src, steps, qstart, qend, qw = chosen
mixed = src["base_audio"].copy();rms = float(np.sqrt(np.mean(question**2)))
gain = min(1.0, .045 / max(rms, 1e-8));question = question * gain
fade = min(160, len(question) // 4);question[:fade] *= np.linspace(0, 1, fade);question[-fade:] *= np.linspace(1, 0, fade)
first = round(qstart * sr);last = first + len(question);mixed[first:last] += question
if np.max(np.abs(mixed)) >= 1.0:
save_json(output, {"id": row["id"], "rejected": "mixed_audio_clipping"});continue
basepath = make_base(model, processor, b, rankroot)
visualpath = rankroot / "visual" / (row["image_sha256"] + ".pt")
if not visualpath.exists():
features = {}
with Image.open(row["image_path"]) as image:
image = ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0))
with torch.inference_mode():
for count in (3, 4, 5):
v = video_inputs(processor, image, count)
grid = v["video_grid_thw"].cuda()
assert grid.tolist() == [[math.ceil(count / 2), 16, 16]], grid
features[str(count)] = model.get_video_features(v["pixel_values_videos"].cuda().to(torch.bfloat16), grid).cpu()
atomic_torch(visualpath, features)
patches = {}
with torch.inference_mode():
for w in range(len(steps)):
start = w * 6400;end = start + 9600
if end <= first or start >= last:
continue
inputs = prepare_qwen_streaming_audio_window(processor, mixed[start:end], first_window=w == 0)
patches[w] = model.get_audio_features(inputs["stream_input_features"][None].cuda(), inputs["stream_feature_attention_mask"][None].cuda()).cpu()
timeline = dict(camera_fps=8., camera_side=224, camera_domain="static_image", question_start=qstart, question_end=qend,
question_voice=str(wav), question_gain=gain, base_body_id=b["id"], original_motion_unchanged=True,
question_text_is_not_model_input=True, answer_not_in_microphone=True, source_seconds=src["source_seconds"],
body_camera_relationship="Independent static camera scene and real G1 motion; no egomotion correspondence claim")
front = args.root / "data/frontend" / (row["id"] + ".pt")
payload = dict(format="static_factorized_v1", base_frontend_path=str(basepath), visual_features_path=str(visualpath),
audio_patches=patches, reply_windows=[qw, qw + 1, qw + 2], answer_target=row["answer"], timeline=timeline)
atomic_torch(front, payload)
from psi.data.seed.swa_frontend import load_swa_frontend
check = load_swa_frontend(front)["packets"]
assert len(check) == len(steps) and sum(bool(p["answer_target"]) for p in check) == 3
assert all(torch.isfinite(p["embeds"]).all() for p in check)
item = {**b, "id": row["id"], "question": row, "frontend_root": str(args.root),
"cohort": "vqav2_static", "camera_domain": "static_image", "swa_has_question": True,
"gt_question_id": row["id"], "base_body_id": b["id"], "timeline": timeline,
"human_agreement": row["human_votes"], "visual_review": visual_review,
"asr_review": audio_review, "frontend_sha256": sha(front)}
save_json(output, item)
if number % 25 == 0:
print("ENCODE", args.rank, number + 1, len(rows), flush=True)
def finalize(args):
old = read(args.existing_plan);all_rows = read(args.root / "image_checked.json")
admitted, rejected = [], Counter()
for row in all_rows:
record = read(args.root / "admitted" / (row["id"] + ".json"))
if "rejected" in record:
rejected[record["rejected"]] += 1
else:
front = args.root / "data/frontend" / (row["id"] + ".pt")
if sha(front) != record["frontend_sha256"]:
raise RuntimeError("Admitted frontend changed")
admitted.append(record)
old_train = sum(r["split"] == "train" for r in old)
new_train = sum(r["split"] == "train" for r in admitted)
if old_train + new_train < 5 * old_train:
raise RuntimeError(f"Fivefold gate not met: old={old_train} new={new_train}; no formal manifest published")
from merge_swa_source_plans import merge
combined = merge([old, admitted])
from validate_swa_manifest import validate_manifest
path = args.root / "source_plan_5x.json"
save_json(path.with_suffix(".pending.json"), combined)
audit = validate_manifest(path.with_suffix(".pending.json"), sample_frontends=8, require_visual=True)
path.with_suffix(".pending.json").replace(path)
save_json(args.root / "admission_audit.json", {"old_train": old_train, "new_train": new_train, "train_expansion": (old_train + new_train) / old_train,
"admitted": len(admitted), "rejections": dict(rejected), "validation": audit, "repeat": 1,
"sources": dict(Counter((r.get("cohort", "legacy") + ":" + r["split"]) for r in combined)),
"qa_license": "VQA Consortium CC BY 4.0; COCO image rights remain those of original Flickr authors",
"image_vs_video": "New VQAv2 static-camera sources complement, not replace, existing dynamic-camera data"})
print("READY", path, old_train + new_train, flush=True)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("phase", choices=("voices", "asr", "review", "encode", "finalize"))
p.add_argument("--root", type=Path, required=True);p.add_argument("--body-plan", type=Path, required=True)
p.add_argument("--base-model", type=Path);p.add_argument("--tts-model", type=Path);p.add_argument("--tts-deps", type=Path)
p.add_argument("--asr-model", type=Path);p.add_argument("--review-model", type=Path);p.add_argument("--existing-plan", type=Path)
p.add_argument("--rank", type=int, default=0);p.add_argument("--world", type=int, default=1)
p.add_argument("--batch", type=int, default=16);p.add_argument("--limit", type=int, default=0)
p.add_argument("--follow", action="store_true", help="Consume published receipts while upstream is still running")
p.add_argument("--upstream-done", type=Path, nargs="*", default=[])
p.add_argument("--poll-seconds", type=float, default=10.)
p.add_argument("--upstream-timeout", type=float, default=14400.)
p.add_argument("--cuda-memory-fraction", type=float, default=1., help="Per-process allocator limit; OOM fails closed")
args = p.parse_args();torch.set_num_threads(2);torch.manual_seed(260918 + args.rank)
compatibility = args.body_plan.parents[1] / "compat/deps310"
if compatibility.is_dir():
sys.path.insert(0, str(compatibility))
if args.phase != "finalize":
torch.cuda.set_device(0)
if not 0 < args.cuda_memory_fraction <= 1:
p.error("--cuda-memory-fraction must be in (0, 1]")
torch.cuda.set_per_process_memory_fraction(args.cuda_memory_fraction, 0)
globals()[args.phase](args)
if __name__ == "__main__":
main()