#!/usr/bin/env python3 """Offline VQAv2 -> spoken, causal SWA sources with independent admission gates. Stages may overlap: voices -> asr, independent image review, then encode when both reviews exist. GPU phases accept rank/world sharding; no distributed model or shared KV is used. All phases are resumable. Frozen inputs are factorized instead of copying the same image and base-motion audio into dozens of windows on disk. Qwen hidden/KV are NOT cached. """ from __future__ import annotations import argparse from collections import Counter from functools import lru_cache import hashlib import json import math from pathlib import Path import re import sys import time import numpy as np from PIL import Image, ImageOps import torch sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) from psi.data.seed.qwen_streaming_audio import prepare_qwen_streaming_audio_window from prepare_swa_vqav2 import save_json, normalized, conservative_question from swa_pipeline_queue import ready_batches SYSTEM = ("You are Qwen, a robot with a live camera and microphone. Camera frames and audio arrive in time order. " "Execute motion commands. Answer visual questions briefly using the current camera view while continuing the requested motion. " "A visual question is not a new motion command. Do not answer before the question is complete.") def sha(path): digest = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(2**20), b""): digest.update(block) return digest.hexdigest() def atomic_torch(path, value): path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_suffix(".partial.pt") torch.save(value, temp) temp.replace(path) def read(path): return json.loads(Path(path).read_text()) def canonical_answer(text): words = normalized(text).split() numbers = dict(zip("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty".split(), map(str, range(21)))) return " ".join(numbers.get(w, w) for w in words if w not in {"a", "an", "the"}) def canonical_speech(text): # Punctuation/case and integer spellings only; never ignore missing words. numbers = dict(zip("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty".split(), map(str, range(21)))) return " ".join(numbers.get(w, w) for w in normalized(text).split()) def voice_jobs(args): bodies = read(args.body_plan) reference = next(b for b in bodies if b["split"] == "train" and b["voice"] == "female_01_en" and len(b["instruction"].split()) >= 4) refsha = sha(reference["reference_audio"]) jobs = {} for row in read(args.root / "candidates.json"): key = hashlib.sha256((refsha + "\0" + row["question"]).encode()).hexdigest()[:32] jobs[row["question"]] = dict(key=key, text=row["question"], reference_audio=reference["reference_audio"], reference_text=reference["instruction"], reference_sha256=refsha) return jobs def shard(rows, args): return rows[args.rank::args.world][:args.limit or None] def work_batches(args, rows, output_path, input_paths): return ready_batches(rows, output_path=output_path, input_paths=input_paths, batch_size=args.batch, follow=args.follow, upstream_done=args.upstream_done, poll_seconds=args.poll_seconds, timeout_seconds=args.upstream_timeout) def voices(args): sys.path.insert(0, str(args.tts_deps)) import soundfile as sf from scipy.signal import resample_poly from qwen_tts import Qwen3TTSModel jobs = sorted(voice_jobs(args).values(), key=lambda r: r["key"]) pending = [r for r in shard(jobs, args) if not (args.root / "voices" / (r["key"] + ".json")).is_file()] if not pending: return model = Qwen3TTSModel.from_pretrained(str(args.tts_model), device_map="cuda:0", dtype=torch.bfloat16, attn_implementation="sdpa") ref = pending[0] prompt = model.create_voice_clone_prompt(ref_audio=ref["reference_audio"], ref_text=ref["reference_text"], x_vector_only_mode=False)[0] pending.sort(key=lambda r: (len(r["text"].split()), r["key"])) start = time.monotonic() for offset in range(0, len(pending), args.batch): batch = pending[offset:offset + args.batch] waves, sr = model.generate_voice_clone(text=[r["text"] for r in batch], language=["English"] * len(batch), voice_clone_prompt=[prompt] * len(batch), do_sample=False, subtalker_dosample=False, max_new_tokens=300) for row, wave in zip(batch, waves): g = math.gcd(int(sr), 16000) wave = resample_poly(np.asarray(wave, np.float32), 16000 // g, int(sr) // g).astype(np.float32) path = args.root / "voices" / (row["key"] + ".wav");path.parent.mkdir(parents=True, exist_ok=True) if not np.isfinite(wave).all() or not .3 <= len(wave) / 16000 <= 15: save_json(path.with_suffix(".json"), {**row, "rejected": "waveform_quality"});continue sf.write(path.with_suffix(".partial.wav"), wave, 16000, subtype="PCM_16") path.with_suffix(".partial.wav").replace(path) save_json(path.with_suffix(".json"), {**row, "seconds": len(wave) / 16000, "wave_sha256": sha(path), "voice": "female_01_en"}) print("TTS", args.rank, offset + len(batch), len(pending), "items/s", round((offset + len(batch)) / (time.monotonic() - start), 2), flush=True) def asr(args): import soundfile as sf from transformers import WhisperForConditionalGeneration, WhisperProcessor jobs = sorted(voice_jobs(args).values(), key=lambda r: r["key"]) processor = WhisperProcessor.from_pretrained(str(args.asr_model), local_files_only=True) model = WhisperForConditionalGeneration.from_pretrained(str(args.asr_model), dtype=torch.float16, local_files_only=True).cuda().eval() model.generation_config.forced_decoder_ids = None rows = shard(jobs, args);processed = 0;started = time.monotonic() batches = work_batches(args, rows, lambda r: args.root / "asr" / (r["key"] + ".json"), lambda r: [args.root / "voices" / (r["key"] + ".json")]) for batch in batches: active, waves = [], [] for row in batch: path = args.root / "voices" / (row["key"] + ".wav") receipt = read(path.with_suffix(".json")) if "rejected" in receipt: save_json(args.root / "asr" / (row["key"] + ".json"), {"accepted": False, "reason": receipt["rejected"]});continue assert sha(path) == receipt["wave_sha256"] wave, sr = sf.read(path, dtype="float32");assert sr == 16000 active.append((row, receipt));waves.append(wave) if waves: inputs = processor(waves, sampling_rate=16000, return_tensors="pt", return_attention_mask=True) inputs = {k: v.cuda().to(dtype=torch.float16 if v.is_floating_point() else v.dtype) for k, v in inputs.items()} with torch.inference_mode(): outputs = model.generate(**inputs, language="en", task="transcribe", max_new_tokens=100) texts = processor.batch_decode(outputs, skip_special_tokens=True) for (row, receipt), heard in zip(active, texts): save_json(args.root / "asr" / (row["key"] + ".json"), { "accepted": canonical_speech(heard) == canonical_speech(row["text"]), "expected": row["text"], "heard": heard, "model": str(args.asr_model), "wave_sha256": receipt["wave_sha256"], "independent_asr": True}) processed += len(batch) print("ASR", args.rank, processed, len(rows), "items/s", round(processed / (time.monotonic() - started), 2), flush=True) def review(args): from transformers import AutoModelForImageTextToText, AutoProcessor rows = shard(read(args.root / "image_checked.json"), args) pending = [r for r in rows if not (args.root / "visual_review" / (r["id"] + ".json")).exists()] if not pending: return processor = AutoProcessor.from_pretrained(str(args.review_model), local_files_only=True) processor.tokenizer.padding_side = "left" model = AutoModelForImageTextToText.from_pretrained(str(args.review_model), dtype=torch.bfloat16, attn_implementation="flash_attention_2", local_files_only=True).cuda().eval() start = time.monotonic() for offset in range(0, len(pending), args.batch): batch = pending[offset:offset + args.batch];images = [];texts = [] for row in batch: with Image.open(row["image_path"]) as image: images.append(ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0))) conversation = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": row["question"] + " Answer with only a short phrase, number, or yes/no."}]}] texts.append(processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)) inputs = processor(text=texts, images=images, padding=True, return_tensors="pt").to("cuda") with torch.inference_mode(): generated = model.generate(**inputs, max_new_tokens=16, do_sample=False) decoded = processor.batch_decode(generated[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True) for row, prediction in zip(batch, decoded): save_json(args.root / "visual_review" / (row["id"] + ".json"), { "accepted": canonical_answer(prediction) == canonical_answer(row["answer"]), "prediction": prediction, "reference": row["answer"], "image_sha256": row["image_sha256"], "model": str(args.review_model), "blind_to_reference": True, "camera_side": 224}) print("REVIEW", args.rank, offset + len(batch), len(pending), "items/s", round((offset + len(batch)) / (time.monotonic() - start), 2), flush=True) @lru_cache(maxsize=32) def body(path): return torch.load(path, weights_only=False, mmap=True) def camera_groups(count): available = .6 + .4 * np.arange(count) times = np.arange(math.floor((available[-1] + 1e-7) * 8) + 1) / 8 ends = np.searchsorted(times, available + 1e-7, side="right") groups = [list(range(int(start), int(end))) for start, end in zip(np.r_[0, ends[:-1]], ends)] return available, groups def video_inputs(processor, image, count): frames = np.repeat(np.asarray(image)[None], count, axis=0) return processor.video_processor(videos=[torch.from_numpy(frames.copy()).permute(0, 3, 1, 2)], do_sample_frames=False, size={"shortest_edge": 224**2, "longest_edge": 224**2}, return_tensors="pt") @torch.inference_mode() def make_base(model, processor, row, root): path = root / "bases" / (row["id"] + ".pt") if path.exists(): return path src = body(row["path"]);steps = [s for s in src["source"]["steps"] if s["update_semantic_cache"]] available, groups = camera_groups(len(steps));conf = model.config prefix = processor.tokenizer.encode('<|im_start|>system\n' + SYSTEM + '<|im_end|>\n<|im_start|>user\n', add_special_tokens=False) packets = [] for i, step in enumerate(steps): feat = step["stream_input_features"][None].cuda();mask = step["stream_feature_attention_mask"][None].cuda() audio = model.get_audio_features(feat, mask) grid = torch.tensor([[math.ceil(len(groups[i]) / 2), 16, 16]], device="cuda") nvideo = int(grid.prod()) // 4 ids = torch.tensor([(prefix if i == 0 else []) + [conf.vision_start_token_id] + [conf.video_token_id] * nvideo + [conf.vision_end_token_id, conf.audio_start_token_id] + [conf.audio_token_id] * len(audio) + [conf.audio_end_token_id]], device="cuda") pos, _ = model.get_rope_index(ids, video_grid_thw=grid, second_per_grids=torch.tensor([.25], device="cuda"), attention_mask=torch.ones_like(ids), audio_seqlens=mask.sum(-1)) emb = model.get_input_embeddings()(ids)[0];amask = ids[0].eq(conf.audio_token_id);vmask = ids[0].eq(conf.video_token_id) emb[amask] = audio.to(emb);emb[vmask] = 0 packets.append(dict(ids=ids[0].cpu(), embeds=emb.cpu(), ordinary_audio=audio.cpu(), audio_mask=amask.cpu(), visual_mask=vmask.cpu(), positions=pos[:, 0].cpu(), next_position=int(pos.max()) + 1, reply_label=0, answer_target="", available_seconds=float(available[i]), video_pts=[j / 8 for j in groups[i]], video_grid=grid[0].cpu(), second_per_grid=.25, camera_indices=groups[i])) atomic_torch(path, {"packets": packets, "system": SYSTEM, "body_path": row["path"], "body_sha256": sha(row["path"]), "base_model": str(model.name_or_path)}) return path def encode(args): import soundfile as sf from transformers import Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration rows = shard(read(args.root / "image_checked.json"), args);jobs = voice_jobs(args) model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(str(args.base_model), dtype=torch.bfloat16, attn_implementation="flash_attention_2", local_files_only=True).cuda().eval().requires_grad_(False) processor = Qwen2_5OmniProcessor.from_pretrained(str(args.base_model), local_files_only=True) bodies = [r for r in read(args.body_plan) if r["voice"] == "female_01_en"] # Per-rank base copies avoid concurrent writes; finalization deduplicates # them only after all workers complete. Their total size is small. rankroot = args.root / f"components_r{args.rank}" batches = work_batches(args, rows, lambda r: args.root / "admitted" / (r["id"] + ".json"), lambda r: ([args.root / "asr" / (jobs[r["question"]]["key"] + ".json"), args.root / "visual_review" / (r["id"] + ".json")] if conservative_question(r["question"]) else [])) for number, row in enumerate(row for batch in batches for row in batch): output = args.root / "admitted" / (row["id"] + ".json") if not conservative_question(row["question"]): save_json(output, {"id": row["id"], "rejected": "not_conservative_visible_state"}) continue if output.exists(): continue job = jobs[row["question"]];audio_review = read(args.root / "asr" / (job["key"] + ".json")) visual_review = read(args.root / "visual_review" / (row["id"] + ".json")) if not audio_review["accepted"] or not visual_review["accepted"]: save_json(output, {"id": row["id"], "rejected": "independent_asr_or_visual_gate"});continue wav = args.root / "voices" / (job["key"] + ".wav") if sha(wav) != audio_review["wave_sha256"] or sha(row["image_path"]) != visual_review["image_sha256"]: raise RuntimeError("Reviewed input content changed") question, sr = sf.read(wav, dtype="float32");assert sr == 16000 candidates = sorted((b for b in bodies if b["split"] == row["split"]), key=lambda b: hashlib.sha256((row["id"] + b["id"]).encode()).hexdigest()) chosen = None for b in candidates: src = body(b["path"]);steps = [s for s in src["source"]["steps"] if s["update_semantic_cache"]] motion_start = next((s["step_index"] / 50 for s in steps if s["control_label"] == 1), None) if motion_start is None: continue qend = .6 + .4 * math.ceil((motion_start + 1.2 + len(question) / sr - .6) / .4) qstart = qend - len(question) / sr;qw = int(round((qend - .6) / .4)) if qend + 2.0 <= src["source_seconds"] and qw + 3 < len(steps) and all(s["control_label"] == 1 for s in steps[max(0, int(qstart / .4)):qw + 3]): chosen = b, src, steps, qstart, qend, qw;break if chosen is None: save_json(output, {"id": row["id"], "rejected": "no_unchanged_body_fits"});continue b, src, steps, qstart, qend, qw = chosen mixed = src["base_audio"].copy();rms = float(np.sqrt(np.mean(question**2))) gain = min(1.0, .045 / max(rms, 1e-8));question = question * gain fade = min(160, len(question) // 4);question[:fade] *= np.linspace(0, 1, fade);question[-fade:] *= np.linspace(1, 0, fade) first = round(qstart * sr);last = first + len(question);mixed[first:last] += question if np.max(np.abs(mixed)) >= 1.0: save_json(output, {"id": row["id"], "rejected": "mixed_audio_clipping"});continue basepath = make_base(model, processor, b, rankroot) visualpath = rankroot / "visual" / (row["image_sha256"] + ".pt") if not visualpath.exists(): features = {} with Image.open(row["image_path"]) as image: image = ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0)) with torch.inference_mode(): for count in (3, 4, 5): v = video_inputs(processor, image, count) grid = v["video_grid_thw"].cuda() assert grid.tolist() == [[math.ceil(count / 2), 16, 16]], grid features[str(count)] = model.get_video_features(v["pixel_values_videos"].cuda().to(torch.bfloat16), grid).cpu() atomic_torch(visualpath, features) patches = {} with torch.inference_mode(): for w in range(len(steps)): start = w * 6400;end = start + 9600 if end <= first or start >= last: continue inputs = prepare_qwen_streaming_audio_window(processor, mixed[start:end], first_window=w == 0) patches[w] = model.get_audio_features(inputs["stream_input_features"][None].cuda(), inputs["stream_feature_attention_mask"][None].cuda()).cpu() timeline = dict(camera_fps=8., camera_side=224, camera_domain="static_image", question_start=qstart, question_end=qend, question_voice=str(wav), question_gain=gain, base_body_id=b["id"], original_motion_unchanged=True, question_text_is_not_model_input=True, answer_not_in_microphone=True, source_seconds=src["source_seconds"], body_camera_relationship="Independent static camera scene and real G1 motion; no egomotion correspondence claim") front = args.root / "data/frontend" / (row["id"] + ".pt") payload = dict(format="static_factorized_v1", base_frontend_path=str(basepath), visual_features_path=str(visualpath), audio_patches=patches, reply_windows=[qw, qw + 1, qw + 2], answer_target=row["answer"], timeline=timeline) atomic_torch(front, payload) from psi.data.seed.swa_frontend import load_swa_frontend check = load_swa_frontend(front)["packets"] assert len(check) == len(steps) and sum(bool(p["answer_target"]) for p in check) == 3 assert all(torch.isfinite(p["embeds"]).all() for p in check) item = {**b, "id": row["id"], "question": row, "frontend_root": str(args.root), "cohort": "vqav2_static", "camera_domain": "static_image", "swa_has_question": True, "gt_question_id": row["id"], "base_body_id": b["id"], "timeline": timeline, "human_agreement": row["human_votes"], "visual_review": visual_review, "asr_review": audio_review, "frontend_sha256": sha(front)} save_json(output, item) if number % 25 == 0: print("ENCODE", args.rank, number + 1, len(rows), flush=True) def finalize(args): old = read(args.existing_plan);all_rows = read(args.root / "image_checked.json") admitted, rejected = [], Counter() for row in all_rows: record = read(args.root / "admitted" / (row["id"] + ".json")) if "rejected" in record: rejected[record["rejected"]] += 1 else: front = args.root / "data/frontend" / (row["id"] + ".pt") if sha(front) != record["frontend_sha256"]: raise RuntimeError("Admitted frontend changed") admitted.append(record) old_train = sum(r["split"] == "train" for r in old) new_train = sum(r["split"] == "train" for r in admitted) if old_train + new_train < 5 * old_train: raise RuntimeError(f"Fivefold gate not met: old={old_train} new={new_train}; no formal manifest published") from merge_swa_source_plans import merge combined = merge([old, admitted]) from validate_swa_manifest import validate_manifest path = args.root / "source_plan_5x.json" save_json(path.with_suffix(".pending.json"), combined) audit = validate_manifest(path.with_suffix(".pending.json"), sample_frontends=8, require_visual=True) path.with_suffix(".pending.json").replace(path) save_json(args.root / "admission_audit.json", {"old_train": old_train, "new_train": new_train, "train_expansion": (old_train + new_train) / old_train, "admitted": len(admitted), "rejections": dict(rejected), "validation": audit, "repeat": 1, "sources": dict(Counter((r.get("cohort", "legacy") + ":" + r["split"]) for r in combined)), "qa_license": "VQA Consortium CC BY 4.0; COCO image rights remain those of original Flickr authors", "image_vs_video": "New VQAv2 static-camera sources complement, not replace, existing dynamic-camera data"}) print("READY", path, old_train + new_train, flush=True) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("phase", choices=("voices", "asr", "review", "encode", "finalize")) p.add_argument("--root", type=Path, required=True);p.add_argument("--body-plan", type=Path, required=True) p.add_argument("--base-model", type=Path);p.add_argument("--tts-model", type=Path);p.add_argument("--tts-deps", type=Path) p.add_argument("--asr-model", type=Path);p.add_argument("--review-model", type=Path);p.add_argument("--existing-plan", type=Path) p.add_argument("--rank", type=int, default=0);p.add_argument("--world", type=int, default=1) p.add_argument("--batch", type=int, default=16);p.add_argument("--limit", type=int, default=0) p.add_argument("--follow", action="store_true", help="Consume published receipts while upstream is still running") p.add_argument("--upstream-done", type=Path, nargs="*", default=[]) p.add_argument("--poll-seconds", type=float, default=10.) p.add_argument("--upstream-timeout", type=float, default=14400.) p.add_argument("--cuda-memory-fraction", type=float, default=1., help="Per-process allocator limit; OOM fails closed") args = p.parse_args();torch.set_num_threads(2);torch.manual_seed(260918 + args.rank) compatibility = args.body_plan.parents[1] / "compat/deps310" if compatibility.is_dir(): sys.path.insert(0, str(compatibility)) if args.phase != "finalize": torch.cuda.set_device(0) if not 0 < args.cuda_memory_fraction <= 1: p.error("--cuda-memory-fraction must be in (0, 1]") torch.cuda.set_per_process_memory_fraction(args.cuda_memory_fraction, 0) globals()[args.phase](args) if __name__ == "__main__": main()