Update README and adapter code
Browse files- README.md +2 -2
- code/hb_deltaflow/deltaflow_check_final.py +78 -0
- code/hb_deltaflow/deltaflow_supervise.py +217 -0
- code/hb_deltaflow/h2_hbaug.py +119 -0
- code/hb_deltaflow/prepare_deltaflow.py +418 -0
- code/hb_deltaflow/prepare_deltaflow_a18.py +199 -0
- code/hb_deltaflow/prepare_deltaflow_h2.py +251 -0
- code/hb_deltaflow/qwen_cache.py +79 -0
- code/hb_deltaflow/train_deltaflow.py +96 -0
- code/hb_deltaflow/verify_cpu.py +381 -0
- code/hb_deltaflow/wholebody_hbaug.py +101 -0
README.md
CHANGED
|
@@ -14,11 +14,11 @@ Delta Flow (ImageWAM / HumanWAM whole-body delta-flow policy) checkpoints traine
|
|
| 14 |
|
| 15 |
| Folder | Task | Robot | Cameras (model input) | Global batch | Proprio augmentation | Checkpoints |
|
| 16 |
|---|---|---|---|---|---|---|
|
| 17 |
-
|
|
| 18 |
|
| 19 |
## What an epoch folder contains
|
| 20 |
|
| 21 |
-
Every `epoch-XXX/`
|
| 22 |
statistics and `protocol.json` (task, data, input/output dimensions, cameras, batch, steps, augmentation, file list).
|
| 23 |
Simple training records are included for reproducibility (training plan, training log, configs, metrics); optimizer /
|
| 24 |
scheduler / RNG / DeepSpeed resume state is not. `code/` holds the adapter code used for training and inference
|
|
|
|
| 14 |
|
| 15 |
| Folder | Task | Robot | Cameras (model input) | Global batch | Proprio augmentation | Checkpoints |
|
| 16 |
|---|---|---|---|---|---|---|
|
| 17 |
+
| `b6-pedal-bin/` | B6 — pick up a chip barrel, step on the pedal bin, drop it in | Unitree G1 (Inspire hands) | ego_view, chest | — | aligned (GR00T A20 recipe) | `epoch-030`, `epoch-040` |
|
| 18 |
|
| 19 |
## What an epoch folder contains
|
| 20 |
|
| 21 |
+
Every checkpoint folder (`epoch-XXX/`, or `step-XXXXXX/` for runs trained for a fixed number of steps) is self-contained for evaluation and deployment: weights, model configuration, normalization
|
| 22 |
statistics and `protocol.json` (task, data, input/output dimensions, cameras, batch, steps, augmentation, file list).
|
| 23 |
Simple training records are included for reproducibility (training plan, training log, configs, metrics); optimizer /
|
| 24 |
scheduler / RNG / DeepSpeed resume state is not. `code/` holds the adapter code used for training and inference
|
code/hb_deltaflow/deltaflow_check_final.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verify a finished Delta-flow 40-epoch run; node 0 then writes R/deltaflow-complete.json.
|
| 2 |
+
|
| 3 |
+
python deltaflow_check_final.py <run_root R> [--write] exit 0 = complete, 1 = not complete
|
| 4 |
+
Checks: 32 finish receipts at global_step = 40 x updates_per_epoch; weights for epochs 10/20/30/40 present and
|
| 5 |
+
non-empty; exactly one training state left (the final step) with the full ZeRO-1 shard set.
|
| 6 |
+
"""
|
| 7 |
+
import argparse
|
| 8 |
+
import json
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import sys
|
| 11 |
+
|
| 12 |
+
WORLD, EPOCHS, EVERY = 32, 40, 10
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def problems(root):
|
| 16 |
+
found = []
|
| 17 |
+
receipt = json.loads((root / 'data-verified.json').read_text())
|
| 18 |
+
u = int(receipt['updates_per_epoch'])
|
| 19 |
+
total = EPOCHS * u
|
| 20 |
+
if receipt.get('fixed_steps'): # 2026-11-12: fixed-step runs (8000 steps, checkpoints 6000 / 8000)
|
| 21 |
+
total = int(receipt['max_steps'])
|
| 22 |
+
out = root / 'run' / 'production'
|
| 23 |
+
for rank in range(WORLD):
|
| 24 |
+
path = out / f'finish-rank-{rank:02d}.json'
|
| 25 |
+
if not path.is_file():
|
| 26 |
+
found.append(f'missing {path.name}')
|
| 27 |
+
continue
|
| 28 |
+
status = json.loads(path.read_text())
|
| 29 |
+
if status.get('rank') != rank or status.get('global_step') != total:
|
| 30 |
+
found.append(f'bad receipt {path.name}: {status}')
|
| 31 |
+
elif status.get('resumed_from_step', 0) == 0 and status.get('optimizer_steps') != total:
|
| 32 |
+
found.append(f'optimizer step count differs in {path.name}: {status}')
|
| 33 |
+
weights = [out / 'checkpoints' / 'weights' / f'step_{e * u:06d}.pt' for e in range(EVERY, EPOCHS + 1, EVERY)]
|
| 34 |
+
if receipt.get('fixed_steps'):
|
| 35 |
+
weights = [out / 'checkpoints' / 'weights' / f'step_{s:06d}.pt' for s in receipt['save_steps']]
|
| 36 |
+
for path in weights:
|
| 37 |
+
if not path.is_file() or path.stat().st_size <= 0:
|
| 38 |
+
found.append(f'missing weights {path.name}')
|
| 39 |
+
state_root = out / 'checkpoints' / 'state'
|
| 40 |
+
states = sorted(p.name for p in state_root.iterdir() if p.is_dir()) if state_root.is_dir() else []
|
| 41 |
+
final = state_root / f'step_{total:06d}'
|
| 42 |
+
if states != [final.name]:
|
| 43 |
+
found.append(f'expected only state {final.name}, found {states}')
|
| 44 |
+
shards = list((final / 'pytorch_model').glob('*_optim_states.pt'))
|
| 45 |
+
model = final / 'pytorch_model' / 'mp_rank_00_model_states.pt'
|
| 46 |
+
if (len(shards) != WORLD or not model.is_file() or not (final / 'trainer_state.json').is_file()
|
| 47 |
+
or any(p.stat().st_size <= 0 for p in [*shards, model] if p.is_file())):
|
| 48 |
+
found.append(f'final training state incomplete ({len(shards)} optimizer shards)')
|
| 49 |
+
return found, dict(weights=[str(p) for p in weights], state=str(final), optimizer_shards=len(shards),
|
| 50 |
+
updates_per_epoch=u, max_steps=total, train_episode_ids='0-99',
|
| 51 |
+
save_steps=receipt.get('save_steps'), fixed_steps=bool(receipt.get('fixed_steps')))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def main():
|
| 55 |
+
parser = argparse.ArgumentParser()
|
| 56 |
+
parser.add_argument('root', type=Path)
|
| 57 |
+
parser.add_argument('--write', action='store_true', help='write deltaflow-complete.json when complete')
|
| 58 |
+
args = parser.parse_args()
|
| 59 |
+
root = args.root.resolve()
|
| 60 |
+
try:
|
| 61 |
+
found, summary = problems(root)
|
| 62 |
+
except (OSError, ValueError, KeyError) as error:
|
| 63 |
+
print(f'DELTAFLOW_INCOMPLETE {root.name}: {type(error).__name__}: {error}', flush=True)
|
| 64 |
+
return 1
|
| 65 |
+
if found:
|
| 66 |
+
print(f'DELTAFLOW_INCOMPLETE {root.name}: ' + '; '.join(found[:8]), flush=True)
|
| 67 |
+
return 1
|
| 68 |
+
if args.write:
|
| 69 |
+
target = root / 'deltaflow-complete.json'
|
| 70 |
+
pending = root / 'deltaflow-complete.json.tmp'
|
| 71 |
+
pending.write_text(json.dumps(summary, indent=2) + '\n')
|
| 72 |
+
pending.replace(target)
|
| 73 |
+
print(f'DELTAFLOW_COMPLETE {root.name}: checkpoint weights + final state verified', flush=True)
|
| 74 |
+
return 0
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
if __name__ == '__main__':
|
| 78 |
+
sys.exit(main())
|
code/hb_deltaflow/deltaflow_supervise.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-node supervisor for one Delta-flow production run on 4 nodes x 8 GPUs (adapted from launch_supervised.py).
|
| 2 |
+
|
| 3 |
+
Usage (called by deltaflow_task.sh on every node, DLC env RANK/WORLD_SIZE/MASTER_ADDR present):
|
| 4 |
+
python deltaflow_supervise.py <run_root R> <session> <port_base>
|
| 5 |
+
Up to 3 attempts. Before each attempt node 0 writes a plan: resume from the newest complete training state in
|
| 6 |
+
R/run/production/checkpoints/state (saved at epoch 10/20/30), or, when none exists, move any partial output aside
|
| 7 |
+
and start fresh. A node that sees a peer fail stops its own launcher; a 30-minute all-idle stall stops all.
|
| 8 |
+
NAS barriers live in R/run/retry/s<session>/attempt-<n>/. Exit 0 only when all four launchers exited 0.
|
| 9 |
+
"""
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
import re
|
| 14 |
+
import signal
|
| 15 |
+
import subprocess
|
| 16 |
+
import sys
|
| 17 |
+
import time
|
| 18 |
+
|
| 19 |
+
HERE = Path(__file__).resolve().parent
|
| 20 |
+
R = Path(sys.argv[1]).resolve()
|
| 21 |
+
SESSION = int(sys.argv[2])
|
| 22 |
+
PORT_BASE = int(sys.argv[3])
|
| 23 |
+
NODE = int(os.environ['RANK'])
|
| 24 |
+
NODES, WORLD, ATTEMPTS = 4, 32, 3
|
| 25 |
+
RUN = R / 'run'
|
| 26 |
+
OUT = RUN / 'production'
|
| 27 |
+
STATE = OUT / 'checkpoints' / 'state'
|
| 28 |
+
CURRENT = {}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def log(*args):
|
| 32 |
+
print(time.strftime('%Y-%m-%d %H:%M:%S'), f'[supervise node {NODE}]', *args, flush=True)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def write(path, text):
|
| 36 |
+
pending = path.with_name(f'{path.name}.tmp{os.getpid()}')
|
| 37 |
+
pending.write_text(text)
|
| 38 |
+
pending.replace(path)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def wait_for_files(paths, seconds=900):
|
| 42 |
+
deadline = time.monotonic() + seconds
|
| 43 |
+
while not all(path.is_file() for path in paths):
|
| 44 |
+
if time.monotonic() >= deadline:
|
| 45 |
+
missing = [str(p) for p in paths if not p.is_file()]
|
| 46 |
+
raise TimeoutError(f'Four nodes did not reach a shared barrier; missing {missing}')
|
| 47 |
+
time.sleep(2)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def gpu_apps():
|
| 51 |
+
return subprocess.check_output(['nvidia-smi', '--query-compute-apps=pid', '--format=csv,noheader'],
|
| 52 |
+
text=True, timeout=20).strip()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def gpu_idle():
|
| 56 |
+
try:
|
| 57 |
+
result = subprocess.check_output(['nvidia-smi', '--query-gpu=utilization.gpu', '--format=csv,noheader,nounits'],
|
| 58 |
+
text=True, timeout=8)
|
| 59 |
+
utilization = [int(line.strip()) for line in result.splitlines()]
|
| 60 |
+
return len(utilization) == 8 and max(utilization) == 0
|
| 61 |
+
except (subprocess.SubprocessError, ValueError):
|
| 62 |
+
return False
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def complete_state(path):
|
| 66 |
+
return ((path / 'trainer_state.json').is_file()
|
| 67 |
+
and (path / 'pytorch_model' / 'mp_rank_00_model_states.pt').is_file()
|
| 68 |
+
and len(list((path / 'pytorch_model').glob('*_optim_states.pt'))) == WORLD)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def newest_state():
|
| 72 |
+
if not STATE.is_dir():
|
| 73 |
+
return None
|
| 74 |
+
found = []
|
| 75 |
+
for path in STATE.iterdir():
|
| 76 |
+
match = re.fullmatch(r'step_(\d+)', path.name)
|
| 77 |
+
if match and path.is_dir() and complete_state(path):
|
| 78 |
+
found.append((int(match.group(1)), path))
|
| 79 |
+
return max(found)[1] if found else None
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def training_idle(attempt_dir, now):
|
| 83 |
+
threshold = now - 1800
|
| 84 |
+
for rank in range(NODES):
|
| 85 |
+
path = attempt_dir / f'gpu-idle-{rank}.json'
|
| 86 |
+
if not path.is_file():
|
| 87 |
+
return False
|
| 88 |
+
report = json.loads(path.read_text())
|
| 89 |
+
if report['sampled'] < now - 90 or report['idle_since'] is None or report['idle_since'] > threshold:
|
| 90 |
+
return False
|
| 91 |
+
activity = list(attempt_dir.glob('node-*.log')) + list((OUT / 'checkpoints').rglob('*'))
|
| 92 |
+
if not activity:
|
| 93 |
+
return False
|
| 94 |
+
for path in activity:
|
| 95 |
+
try:
|
| 96 |
+
if path.stat().st_mtime > threshold:
|
| 97 |
+
return False
|
| 98 |
+
except FileNotFoundError:
|
| 99 |
+
return False
|
| 100 |
+
return True
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def wait_for_process(process, peers, attempt_dir):
|
| 104 |
+
next_sample, idle_since = 0.0, None
|
| 105 |
+
stalled = attempt_dir / 'training-stalled'
|
| 106 |
+
while process.poll() is None:
|
| 107 |
+
if time.monotonic() >= next_sample:
|
| 108 |
+
now = time.time()
|
| 109 |
+
idle_since = (idle_since if idle_since is not None else now) if gpu_idle() else None
|
| 110 |
+
write(attempt_dir / f'gpu-idle-{NODE}.json', json.dumps({'sampled': now, 'idle_since': idle_since}))
|
| 111 |
+
next_sample = time.monotonic() + 30
|
| 112 |
+
if NODE == 0 and training_idle(attempt_dir, now):
|
| 113 |
+
write(stalled, 'All four nodes idle and no log/checkpoint progress for 30 minutes.\n')
|
| 114 |
+
if stalled.exists() or any(path.is_file() and path.read_text().strip() != '0' for path in peers):
|
| 115 |
+
log('peer failure or stall detected; stopping local launcher')
|
| 116 |
+
try:
|
| 117 |
+
os.killpg(process.pid, signal.SIGTERM)
|
| 118 |
+
except ProcessLookupError:
|
| 119 |
+
pass
|
| 120 |
+
try:
|
| 121 |
+
return process.wait(timeout=120)
|
| 122 |
+
except subprocess.TimeoutExpired:
|
| 123 |
+
os.killpg(process.pid, signal.SIGKILL)
|
| 124 |
+
return process.wait(timeout=60)
|
| 125 |
+
time.sleep(2)
|
| 126 |
+
return process.returncode
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def release_gpus():
|
| 130 |
+
deadline = time.monotonic() + 300
|
| 131 |
+
while gpu_apps():
|
| 132 |
+
if time.monotonic() >= deadline:
|
| 133 |
+
raise TimeoutError('GPU processes did not exit after a failed attempt')
|
| 134 |
+
if time.monotonic() >= deadline - 180:
|
| 135 |
+
subprocess.call(['pkill', '-9', '-f', 'scripts/wholebody/train_deltaflow.py'])
|
| 136 |
+
time.sleep(5)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def plan_attempt(attempt, attempt_dir):
|
| 140 |
+
"""Node 0 only: decide resume vs fresh start (all nodes are idle at this point)."""
|
| 141 |
+
state = newest_state()
|
| 142 |
+
if state is not None:
|
| 143 |
+
plan = {'resume': str(state), 'step': int(state.name.split('_')[1])}
|
| 144 |
+
else:
|
| 145 |
+
if OUT.exists():
|
| 146 |
+
aside = RUN / f'discarded-s{SESSION}-a{attempt}-{time.strftime("%Y%m%dT%H%M%S")}'
|
| 147 |
+
OUT.rename(aside)
|
| 148 |
+
log('no complete training state; partial output moved to', aside)
|
| 149 |
+
plan = {'resume': None, 'step': 0}
|
| 150 |
+
write(attempt_dir / 'plan.json', json.dumps(plan) + '\n')
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def run():
|
| 154 |
+
if NODE not in range(NODES):
|
| 155 |
+
raise ValueError('Unexpected node rank')
|
| 156 |
+
session_root = RUN / 'retry' / f's{SESSION}'
|
| 157 |
+
for attempt in range(1, ATTEMPTS + 1):
|
| 158 |
+
attempt_dir = session_root / f'attempt-{attempt}'
|
| 159 |
+
attempt_dir.mkdir(parents=True, exist_ok=True)
|
| 160 |
+
ready = [attempt_dir / f'ready-{rank}' for rank in range(NODES)]
|
| 161 |
+
done = [attempt_dir / f'exit-{rank}' for rank in range(NODES)]
|
| 162 |
+
CURRENT['exit'] = done[NODE]
|
| 163 |
+
if ready[NODE].exists():
|
| 164 |
+
raise RuntimeError(f'This node already joined {attempt_dir} (session counters out of sync?)')
|
| 165 |
+
deadline = time.monotonic() + 180
|
| 166 |
+
while gpu_apps():
|
| 167 |
+
if time.monotonic() >= deadline:
|
| 168 |
+
raise RuntimeError('GPUs busy before launch')
|
| 169 |
+
time.sleep(5)
|
| 170 |
+
write(ready[NODE], f'{os.getpid()}\n')
|
| 171 |
+
wait_for_files(ready)
|
| 172 |
+
if NODE == 0:
|
| 173 |
+
plan_attempt(attempt, attempt_dir)
|
| 174 |
+
wait_for_files([attempt_dir / 'plan.json'], seconds=300)
|
| 175 |
+
plan = json.loads((attempt_dir / 'plan.json').read_text())
|
| 176 |
+
if any(path.exists() for path in done):
|
| 177 |
+
raise RuntimeError('A result exists before the attempt began')
|
| 178 |
+
port = PORT_BASE + 3 * ((SESSION - 1) % 3) + attempt - 1
|
| 179 |
+
log(f'attempt {attempt} port {port} resume={plan["resume"]}')
|
| 180 |
+
with (attempt_dir / f'node-{NODE}.log').open('ab') as stream:
|
| 181 |
+
process = subprocess.Popen(['bash', str(HERE / 'deltaflow_attempt.sh'), str(R), str(port),
|
| 182 |
+
plan['resume'] or 'none'],
|
| 183 |
+
stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT,
|
| 184 |
+
start_new_session=True)
|
| 185 |
+
status = wait_for_process(process, [path for i, path in enumerate(done) if i != NODE], attempt_dir)
|
| 186 |
+
log(f'attempt {attempt} local launcher exit {status}')
|
| 187 |
+
write(done[NODE], f'{status}\n')
|
| 188 |
+
wait_for_files(done, seconds=1800)
|
| 189 |
+
if all(path.read_text().strip() == '0' for path in done):
|
| 190 |
+
return 0
|
| 191 |
+
if attempt < ATTEMPTS:
|
| 192 |
+
release_gpus()
|
| 193 |
+
drained = [attempt_dir / f'gpu-released-{rank}' for rank in range(NODES)]
|
| 194 |
+
write(drained[NODE], 'idle\n')
|
| 195 |
+
wait_for_files(drained, seconds=600)
|
| 196 |
+
return 1
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def main():
|
| 200 |
+
log_dir = RUN / 'node-logs'
|
| 201 |
+
log_dir.mkdir(parents=True, exist_ok=True)
|
| 202 |
+
try:
|
| 203 |
+
result = run()
|
| 204 |
+
except Exception as error: # noqa: BLE001 - report every failure to the NAS and exit nonzero
|
| 205 |
+
write(log_dir / f's{SESSION}-supervisor-error-{NODE}.json',
|
| 206 |
+
json.dumps({'type': type(error).__name__, 'detail': str(error)}) + '\n')
|
| 207 |
+
log('ERROR', type(error).__name__, error)
|
| 208 |
+
result = 1
|
| 209 |
+
exit_file = CURRENT.get('exit')
|
| 210 |
+
if exit_file is not None and not exit_file.exists():
|
| 211 |
+
write(exit_file, '1\n') # lets the peers stop their launchers instead of waiting for timeouts
|
| 212 |
+
write(log_dir / f's{SESSION}-node-{NODE}.exit', f'{result}\n')
|
| 213 |
+
return result
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
if __name__ == '__main__':
|
| 217 |
+
sys.exit(main())
|
code/hb_deltaflow/h2_hbaug.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A18 (H2 + Sharpa hands) Delta-flow loader with augmentation spec v2. NO joint velocity.
|
| 2 |
+
|
| 3 |
+
Installed by prepare_deltaflow_a18.py as imagewam/datasets/h2_hbaug.py in the H2 code copy (HumanWAM-h2), next to
|
| 4 |
+
an unchanged copy of $G/hb_groot/hb_aug_v2.py. Builds on H2PourDataset of a18-h2-pour-from-scratch-20261108:
|
| 5 |
+
same cameras (head + chest; the collection also records wrist and third_person, unused as before), same
|
| 6 |
+
normalization file format (q01/q99 of state78 = 75 joints + gravity3, action75 = body_target31 + hands44),
|
| 7 |
+
same 16-step action chunk / history and t+16 image target. Changes:
|
| 8 |
+
- visual: hb_aug_v2.augment on [T=2, V=2, 3, 224, 224] (15 effects, one boosted to 1.5x per sample, colour shared,
|
| 9 |
+
rotation/light/WB per camera on t and t+16, sensor/optics only on the observation frame t);
|
| 10 |
+
- proprio (2026-11-12, aligned with GR00T A20; H2 body = state[0:31] incl. waist and head):
|
| 11 |
+
hb_aug_v2.perturb_joints per sample (+-2 deg bias + N(0, 0.25 deg) noise clipped at 3 sigma), no delay,
|
| 12 |
+
no history drop; hands, gravity and every label untouched.
|
| 13 |
+
Model input: proprio[16, 78] = normalized(75 joints + gravity 3), exactly the state of the original H2 model
|
| 14 |
+
(h2_pour_delta.py, HF g1-basket-transfer-imagewam-deltaflow76 h2-a18-pour-water/epoch-040: proprio_dim 78).
|
| 15 |
+
User rule 2026-11-10: no joint-velocity channels in any training; any velocity argument is an error.
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import math
|
| 20 |
+
import os
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import torch
|
| 24 |
+
|
| 25 |
+
from . import hb_aug_v2
|
| 26 |
+
from .h2_pour import ACTION_DIM, CAMERAS, STATE_DIM, H2PourDataset
|
| 27 |
+
from .wholebody import window_indices
|
| 28 |
+
|
| 29 |
+
BODY = 31
|
| 30 |
+
SPEC_VERSION = "hb-aug-v2-20261109"
|
| 31 |
+
# 2026-11-12 (user): proprio aligned with GR00T's A20 basket-transfer training (hb_groot/hb_launch.py):
|
| 32 |
+
# hb_aug_v2.perturb_joints on the 31 body joints per sample, no delay, no history drop.
|
| 33 |
+
PROPRIO_SPEC = {"enabled": True, "joint_bias_degrees": 2.0, "joint_noise_std_degrees": 0.25,
|
| 34 |
+
"delay_probability": 0.0, "max_delay_control_cycles": 0, "control_hz": 30, "recording_hz": 30,
|
| 35 |
+
"history_drop_probability": 0.0, "bias_scope": "per_sample",
|
| 36 |
+
"implementation": "hb_aug_v2.perturb_joints"}
|
| 37 |
+
if STATE_DIM != 78:
|
| 38 |
+
raise ImportError(f"H2 state must be 78-D (75 joints + gravity 3), h2_pour says {STATE_DIM}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class H2HBAugV2Dataset(H2PourDataset):
|
| 42 |
+
def __init__(self, *, image_augmentation, proprio_augmentation, **kwargs):
|
| 43 |
+
velocity_args = sorted(k for k in kwargs if "velocity" in k)
|
| 44 |
+
if velocity_args:
|
| 45 |
+
raise ValueError(f"Joint velocity is not allowed in the model input (got {velocity_args})")
|
| 46 |
+
image_policy = dict(image_augmentation)
|
| 47 |
+
proprio_policy = dict(proprio_augmentation)
|
| 48 |
+
if hb_aug_v2.VERSION != SPEC_VERSION or image_policy.get("version") != SPEC_VERSION:
|
| 49 |
+
raise ValueError(f"Expected augmentation {SPEC_VERSION}, got module {hb_aug_v2.VERSION} / "
|
| 50 |
+
f"config {image_policy.get('version')}")
|
| 51 |
+
if image_policy.get("enabled") is not True or "seed" not in image_policy:
|
| 52 |
+
raise ValueError("Augmentation v2 must be enabled and seeded")
|
| 53 |
+
for key, value in PROPRIO_SPEC.items():
|
| 54 |
+
if proprio_policy.get(key) != value:
|
| 55 |
+
raise ValueError(f"proprio_augmentation.{key}={proprio_policy.get(key)!r}, spec v2 needs {value!r}")
|
| 56 |
+
if list(proprio_policy.get("affected_angle_channels", [])) != list(range(BODY)) or "seed" not in proprio_policy:
|
| 57 |
+
raise ValueError("Bias/noise must hit exactly the 31 H2 body joints (state[0:31])")
|
| 58 |
+
if list(proprio_policy.get("velocity_channels") or []):
|
| 59 |
+
raise ValueError("velocity_channels must be empty (no joint velocity)")
|
| 60 |
+
# The parent's own (spec v1) image augmentation stays disabled; hb_aug_v2 replaces it below.
|
| 61 |
+
super().__init__(image_augmentation=None, **kwargs)
|
| 62 |
+
self.image_policy, self.proprio_policy = image_policy, proprio_policy
|
| 63 |
+
self._aug_rng, self._aug_key = None, None
|
| 64 |
+
self.last_augmentation = {}
|
| 65 |
+
|
| 66 |
+
def _rng(self):
|
| 67 |
+
worker = torch.utils.data.get_worker_info()
|
| 68 |
+
worker_id = worker.id if worker else 0
|
| 69 |
+
rank = int(os.environ.get("RANK", "0"))
|
| 70 |
+
initial = int(torch.initial_seed())
|
| 71 |
+
key = (os.getpid(), rank, worker_id, initial)
|
| 72 |
+
if key != self._aug_key:
|
| 73 |
+
self._aug_rng = np.random.default_rng(np.random.SeedSequence(
|
| 74 |
+
[int(self.image_policy["seed"]), rank, worker_id, initial % (2 ** 32), initial // (2 ** 32)]))
|
| 75 |
+
self._aug_key = key
|
| 76 |
+
return self._aug_rng
|
| 77 |
+
|
| 78 |
+
def augment_numeric(self, episode, anchor, state, past, pad, rng):
|
| 79 |
+
raw = state[anchor].copy() # 75 joints + gravity 3, radians; no delay
|
| 80 |
+
before = raw[:BODY].astype(np.float64)
|
| 81 |
+
raw[:BODY] = hb_aug_v2.perturb_joints(raw[:BODY], rng) # per-sample bias +-2 deg + noise 0.25 deg
|
| 82 |
+
proprio = self.normalize(raw, "state").astype(np.float32)
|
| 83 |
+
if proprio.shape != (STATE_DIM,):
|
| 84 |
+
raise ValueError(f"proprio must be {STATE_DIM}-D (no velocity), got {proprio.shape}")
|
| 85 |
+
past[pad] = 0 # unchanged history, no drop
|
| 86 |
+
info = dict(delay_recorded_frames=0, state_frame=anchor, anchor_frame=anchor, history_drop_rows=[],
|
| 87 |
+
joint_perturbation_degrees=np.degrees(raw[:BODY].astype(np.float64) - before).tolist())
|
| 88 |
+
return proprio, past, pad, info
|
| 89 |
+
|
| 90 |
+
def __getitem__(self, index):
|
| 91 |
+
if index < 0 or index >= len(self):
|
| 92 |
+
raise IndexError(index)
|
| 93 |
+
ep_position = int(np.searchsorted(self.ends, index, side="right"))
|
| 94 |
+
episode = self.episode_ids[ep_position]
|
| 95 |
+
anchor = int(index - (self.ends[ep_position - 1] if ep_position else 0))
|
| 96 |
+
state, action = self.episodes[episode]
|
| 97 |
+
window = window_indices(anchor, len(action), 16, 16)
|
| 98 |
+
views = torch.stack([self._frames(episode, c, window["image"].tolist()) for c in CAMERAS], dim=1)
|
| 99 |
+
rng = self._rng()
|
| 100 |
+
observation = torch.tensor([True, False]) # t+16 is the supervised image target
|
| 101 |
+
views, visual = hb_aug_v2.augment(views, rng, obs_mask=observation)
|
| 102 |
+
video = (torch.cat(list(views.unbind(1)), dim=-1) * 2 - 1).permute(1, 0, 2, 3).contiguous()
|
| 103 |
+
future = self.normalize(action[window["action"]], "action")
|
| 104 |
+
past = self.normalize(action[window["past_action"]], "action")
|
| 105 |
+
if window["past_action_is_pad"].all():
|
| 106 |
+
past = np.zeros_like(past)
|
| 107 |
+
proprio, past, pad, numeric = self.augment_numeric(
|
| 108 |
+
episode, anchor, state, torch.from_numpy(past), torch.from_numpy(window["past_action_is_pad"].copy()), rng)
|
| 109 |
+
self.last_augmentation = dict(visual=visual, numeric=numeric)
|
| 110 |
+
return dict(video=video, action=torch.from_numpy(future), past_action=past,
|
| 111 |
+
proprio=torch.from_numpy(np.repeat(proprio[None], 16, axis=0)),
|
| 112 |
+
action_is_pad=torch.from_numpy(window["action_is_pad"]), past_action_is_pad=pad,
|
| 113 |
+
image_is_pad=torch.from_numpy(window["image_is_pad"]),
|
| 114 |
+
proprio_is_pad=torch.zeros(16, dtype=torch.bool),
|
| 115 |
+
action_dim_is_pad=torch.zeros(ACTION_DIM, dtype=torch.bool),
|
| 116 |
+
proprio_dim_is_pad=torch.zeros(STATE_DIM, dtype=torch.bool),
|
| 117 |
+
text_hidden_states=self.text_hidden_states, text_attention_mask=self.text_attention_mask,
|
| 118 |
+
prompt=self.prompt, instruction=self.prompt, idx=int(index), episode_index=int(episode),
|
| 119 |
+
frame_index=anchor, task_index=0, dataset_index=0)
|
code/hb_deltaflow/prepare_deltaflow.py
ADDED
|
@@ -0,0 +1,418 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One-time CPU preparation of the ImageWAM Delta-flow 40-epoch runs (B6 / A3 / A7, 100 episodes each, aug v2).
|
| 2 |
+
|
| 3 |
+
NO joint velocity (user rule 2026-11-10): the model input is 44-D (body 29 + hands 12 + gravity 3), exactly like the
|
| 4 |
+
reference HF humanoid-benchmark/b6-imagewam-deltaflow76 real30-from-sim60-20261108 (proprio_dim = action_state_dim
|
| 5 |
+
= 44, velocity_channels []). The previous run's YAML is a 73-D (velocity) config: its velocity keys are removed and
|
| 6 |
+
the model dims are set to 44; no velocity cache or statistics are produced, and no_velocity() rejects any leftover.
|
| 7 |
+
|
| 8 |
+
Run on node 0 (no GPU needed), after deploying this directory to $G/hb_deltaflow:
|
| 9 |
+
$P/venv/bin/python $G/hb_deltaflow/prepare_deltaflow.py # all of b6 a3 a7 a18
|
| 10 |
+
$P/venv/bin/python $G/hb_deltaflow/prepare_deltaflow.py b6 # one task
|
| 11 |
+
(a18 = H2 is delegated to prepare_deltaflow_a18.py in a separate interpreter; see that file)
|
| 12 |
+
Creates (never touches the old experiments or the datasets):
|
| 13 |
+
$G/deltaflow-runs/HumanWAM/ code copy of the last B6 Delta-flow run + 100-episode loader patch,
|
| 14 |
+
hb_aug_v2.py, wholebody_hbaug.py, train_deltaflow.py
|
| 15 |
+
$G/deltaflow-runs/<task>/{split-manifest.json, stats/, qwen-text/, run/production.yaml, data-verified.json,
|
| 16 |
+
prepared.ok}
|
| 17 |
+
Idempotent: finished tasks (prepared.ok) are skipped; a half-prepared task directory is renamed aside and redone.
|
| 18 |
+
"""
|
| 19 |
+
import hashlib
|
| 20 |
+
import json
|
| 21 |
+
import math
|
| 22 |
+
import os
|
| 23 |
+
import shutil
|
| 24 |
+
import sys
|
| 25 |
+
import time
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
import numpy as np
|
| 29 |
+
import pyarrow.parquet as pq
|
| 30 |
+
import yaml
|
| 31 |
+
|
| 32 |
+
H = Path(__file__).resolve().parent # $G/hb_deltaflow
|
| 33 |
+
G = H.parent # $P/policies-20261109
|
| 34 |
+
P = G.parent # preflight-20261104T1648Z
|
| 35 |
+
OLD_B6 = P / 'b6-v5t7-ready-start-60ep-20261109'
|
| 36 |
+
OLD_A = P / 'a3-a7-20261109'
|
| 37 |
+
SRC_CODE = OLD_B6 / 'HumanWAM'
|
| 38 |
+
AUG_SRC = G / 'hb_groot' / 'hb_aug_v2.py'
|
| 39 |
+
# The B6 60-epoch copy's wholebody_delta.py hard-codes the 73-D (44 + 29 joint velocity) state; the 44-D reference run
|
| 40 |
+
# real30-from-sim60-20261108 (HF humanoid-benchmark/b6-imagewam-deltaflow76) used the same file with 44 instead.
|
| 41 |
+
REF_DELTA = P / 'real30-from-sim60-20261108' / 'HumanWAM' / 'src/imagewam/wholebody_delta.py'
|
| 42 |
+
REF_DELTA_SHA256 = '523244263496679f7046035f5df3d34590b7086e25173938d8a175dad963b9f3'
|
| 43 |
+
ROOT = G / 'deltaflow-runs'
|
| 44 |
+
CODE = ROOT / 'HumanWAM'
|
| 45 |
+
|
| 46 |
+
EPISODES, EPOCHS, SAVE_EVERY_EPOCHS, WARMUP_EPOCHS = 100, 40, 10, 2
|
| 47 |
+
WORLD, MICRO = 32, 8
|
| 48 |
+
# 2026-11-12 user: fixed 8000 optimizer steps (no epochs); checkpoints every 2000 steps counted back from the last
|
| 49 |
+
# step, only the last two kept (the last one with its resume state); warmup 5 % (the former 2 of 40 epochs).
|
| 50 |
+
FIXED_STEPS, CKPT_EVERY, KEEP_CKPTS, WARMUP_FRAC = 8000, 2000, 2, 0.05
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def fixed_schedule(u):
|
| 54 |
+
total = FIXED_STEPS
|
| 55 |
+
save_steps = [total - CKPT_EVERY * k for k in range(KEEP_CKPTS - 1, -1, -1)]
|
| 56 |
+
return total, save_steps, round(WARMUP_FRAC * total), math.ceil(total / u)
|
| 57 |
+
DATASET_TARGET = 'imagewam.datasets.wholebody_hbaug.WholeBodyHBAugV2LeRobotDataset'
|
| 58 |
+
AUG_VERSION = 'hb-aug-v2-20261109'
|
| 59 |
+
AUG_SHA256 = '9f7713cea4f846efcfc924234267ced48210357f419d67915c10b504316bedde'
|
| 60 |
+
TASKS = {
|
| 61 |
+
'b6': dict(data=OLD_B6 / 'data/deliveries/g1/b6-v5t7-ready-start-20261109', old=OLD_B6 / 'train',
|
| 62 |
+
control_hz=50, total_episodes=149),
|
| 63 |
+
'a3': dict(data=OLD_A / 'a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45', old=OLD_A / 'a3',
|
| 64 |
+
control_hz=30, total_episodes=100),
|
| 65 |
+
'a7': dict(data=OLD_A / 'a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c', old=OLD_A / 'a7',
|
| 66 |
+
control_hz=30, total_episodes=100),
|
| 67 |
+
# 2026-11-12: tasks without a previous Delta-flow experiment. Template config = the A3 run (same G1 model / loader);
|
| 68 |
+
# split manifest and instruction cache are generated here. Datasets are aligned with basket-transfer-expert-v1
|
| 69 |
+
# (identical 41 features / modality, 30 FPS); the fixed first 100 episodes are trained, later ones shelved.
|
| 70 |
+
'trolley': dict(data=P / 'trolley-push-expert-v1-20261110/data/trolley_push', old=None, control_hz=50,
|
| 71 |
+
total_episodes=None, source='trolley-push-expert-v1 @ef9024a'),
|
| 72 |
+
'basket': dict(data=P / 'basket-current-dynamics-100-v21-b0ff522/data', old=None, control_hz=50,
|
| 73 |
+
total_episodes=None, source='basket-transfer-expert-v1 @b0ff522 (current-dynamics-100-v21)'),
|
| 74 |
+
'b7': dict(data=P / 'b7-black-clothes-hamper-v1-1bb002a/data', old=None, control_hz=50,
|
| 75 |
+
total_episodes=None, source='b7-black-clothes-hamper-expert-v1 @1bb002a'),
|
| 76 |
+
'a6': dict(data=P / 'a6-door-pull-v1-cf3ee9b/data/door_pull', old=None, control_hz=50,
|
| 77 |
+
total_episodes=None, source='door-pull-expert-v1 @cf3ee9b'),
|
| 78 |
+
'c6': dict(data=P / 'c6-bathroom-door-fetch-v1-3afba69/data', old=None, control_hz=50,
|
| 79 |
+
total_episodes=None, source='c6-bathroom-door-fetch-expert-v1 @3afba69e'),
|
| 80 |
+
}
|
| 81 |
+
TEMPLATE = OLD_A / 'a3'
|
| 82 |
+
QWEN3 = P / 'hf-inputs/Qwen3-4B'
|
| 83 |
+
FLUX2_SRC = P / 'HumanWAM/third_party/flux2'
|
| 84 |
+
H2_TASKS = ('a18', 'a17')
|
| 85 |
+
# 2026-11-12 (user): every model aligned with GR00T's A20 basket-transfer training (hb_groot/hb_launch.py).
|
| 86 |
+
ALIGNED_PROPRIO = dict(delay_probability=0.0, max_delay_control_cycles=0, history_drop_probability=0.0,
|
| 87 |
+
bias_scope='per_sample', implementation='hb_aug_v2.perturb_joints',
|
| 88 |
+
aligned_with='GR00T A20 basket-transfer recipe (hb_groot/hb_launch.py), user 2026-11-12')
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def new_manifest(task, spec, meta, ids):
|
| 92 |
+
D = spec['data']
|
| 93 |
+
names = meta['info']['features']['observation.state'].get('names') or []
|
| 94 |
+
hashes = {rel: sha(D / rel) for rel in ('meta/info.json', 'meta/episodes.jsonl', 'meta/source_episodes.jsonl')
|
| 95 |
+
if (D / rel).is_file()}
|
| 96 |
+
stem = spec['source'].split(' ')[0]
|
| 97 |
+
return dict(train_episode_ids=ids[:EPISODES], validation_episode_ids=ids[EPISODES:], seed=42,
|
| 98 |
+
provenance_reviewed=True,
|
| 99 |
+
provenance_note=(f'generated 2026-11-12 by prepare_deltaflow.py for HF {spec["source"]}; one source '
|
| 100 |
+
'group per episode; dataset format checked against basket-transfer-expert-v1'),
|
| 101 |
+
split_basis='', source_group_by_episode={str(i): f'{stem}-{i:06d}' for i in ids},
|
| 102 |
+
expected_body_joint_names=list(names[:29]), source_metadata_sha256=hashes)
|
| 103 |
+
# Files of the last B6 Delta-flow code copy that this package builds on (hashes of the reviewed copy).
|
| 104 |
+
SOURCE_SHA256 = {
|
| 105 |
+
'src/imagewam/datasets/wholebody.py': 'e6460388758187180a28515cb15b666d948387b1b6aff7b026baa2f2e9402ccc',
|
| 106 |
+
'src/imagewam/datasets/wholebody_velocity.py': 'a5788736652772aea2fcf305b47ca6a6c11e1ea798fe8e2cf7bcdd1a242b23ea',
|
| 107 |
+
'src/imagewam/datasets/wholebody_augmented_v3.py': '0e34ea3a71dde2f05bd4bd314ec97787bf3d1809c1cedc3dff5e8610b7723366',
|
| 108 |
+
'src/imagewam/datasets/wholebody_augmented.py': '641bb399f54206fe1c923c4b24fd1b9797787616154915523f10a9480daf7da3',
|
| 109 |
+
'src/imagewam/trainer.py': 'cdcd48e09d1d7b81cae3a829bd415ed4b53ff1b290aacac1ce2217876f475b1a',
|
| 110 |
+
'src/imagewam/runtime.py': '3ebc45f61b0c22fb659012a7d76bea671a42e1e528b5c917ae1e8827e88b890b',
|
| 111 |
+
'src/imagewam/wholebody_delta.py': '20135b5a6cc0378c76d3eb8d958cbc4c3aeb773909b1453081f4fcd6238d36ea',
|
| 112 |
+
'scripts/wholebody/train_a3a7.py': '2cf154f91de516f96da9b93f4b8e235fab13344461b2aeb5d048e4f78a63dec8',
|
| 113 |
+
'scripts/accelerate_configs/accelerate_zero1_ds.yaml': '5b9e24f38c199fdff6bbd17981d86e2624f855f707660094c99f711efc8e0833',
|
| 114 |
+
}
|
| 115 |
+
# wholebody.py of the B6 run accepts exactly 149/0 episodes; make it accept exactly episodes 0-99 for training,
|
| 116 |
+
# with any later episodes listed as shelved (validation_episode_ids, never loaded for training or statistics).
|
| 117 |
+
LOADER_PATCHES = [
|
| 118 |
+
('STATS_SCHEMA = "wholebody-state41to44-action76-chunk16-train149val0-v1"',
|
| 119 |
+
'STATS_SCHEMA = "wholebody-state41to44-action76-chunk16-first100-v1"', 1),
|
| 120 |
+
(' if (len(train) != 149 or len(val) != 0 or train != sorted(set(train))\n'
|
| 121 |
+
' or val != sorted(set(val)) or set(train) & set(val)\n'
|
| 122 |
+
' or sorted(train + val) != ids):\n'
|
| 123 |
+
' raise ValueError("Require exact disjoint 149/0 episode partition of 149 episodes")\n',
|
| 124 |
+
' if (train != list(range(100)) or val != sorted(set(val)) or set(train) & set(val)\n'
|
| 125 |
+
' or sorted(train + val) != ids):\n'
|
| 126 |
+
' raise ValueError("Require training episodes exactly 0-99; later episodes may only be listed as shelved")\n',
|
| 127 |
+
1),
|
| 128 |
+
(' if train_split_mode != "all_149" or len(meta["train"]) != 149 or len(meta["validation"]) != 0:\n'
|
| 129 |
+
' raise ValueError("Require reviewed all-149 training partition")\n',
|
| 130 |
+
' if train_split_mode != "first_100" or list(meta["train"]) != list(range(100)):\n'
|
| 131 |
+
' raise ValueError("Require the fixed first-100 training partition (episodes 0-99)")\n', 1),
|
| 132 |
+
('"all_149"', '"first_100"', 7),
|
| 133 |
+
('if not np.allclose(columns["timestamp"], np.arange(n) / 30.0, rtol=0, atol=3e-6):',
|
| 134 |
+
'if not np.allclose(columns["timestamp"], np.arange(n) / 30.0, rtol=1e-7, atol=3e-6): # float32 timestamps of long (75 s) C6 episodes', 1),
|
| 135 |
+
]
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def sha(path):
|
| 139 |
+
digest = hashlib.sha256()
|
| 140 |
+
with Path(path).open('rb') as stream:
|
| 141 |
+
for block in iter(lambda: stream.read(16 << 20), b''):
|
| 142 |
+
digest.update(block)
|
| 143 |
+
return digest.hexdigest()
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def dump(path, value):
|
| 147 |
+
Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def log(*args):
|
| 151 |
+
print(time.strftime('%H:%M:%S'), *args, flush=True)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def augmentation_config():
|
| 155 |
+
return dict(
|
| 156 |
+
version=AUG_VERSION, enabled=True, seed=20261110, spec='project CLAUDE.md augmentation default spec v2 (2026-11-09)',
|
| 157 |
+
implementation='imagewam/datasets/hb_aug_v2.py = unchanged copy of $G/hb_groot/hb_aug_v2.py',
|
| 158 |
+
implementation_sha256=AUG_SHA256, boost='one of 15 effects uniformly per sample, magnitude x1.5',
|
| 159 |
+
colour='exposure/contrast/saturation/hue/gamma/temperature shared by both cameras and both times',
|
| 160 |
+
per_camera='rotation+crop, light/shadow patches, single-channel white balance; same on t and t+16',
|
| 161 |
+
observation_only='noise, gaussian blur, JPEG, motion blur, sharpen, defocus (never on the t+16 target)',
|
| 162 |
+
native_model_image_augmentation='none', action_label_noise=False, state_label_noise=False)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def build_code():
|
| 166 |
+
record = CODE / 'deltaflow-code.json'
|
| 167 |
+
if record.is_file():
|
| 168 |
+
for rel, digest in json.loads(record.read_text())['sha256'].items():
|
| 169 |
+
if sha(CODE / rel) != digest:
|
| 170 |
+
raise SystemExit(f'{CODE / rel} changed after preparation; rename {CODE} aside and rerun')
|
| 171 |
+
log('code copy present and unchanged:', CODE)
|
| 172 |
+
return
|
| 173 |
+
if CODE.exists():
|
| 174 |
+
raise SystemExit(f'{CODE} exists without deltaflow-code.json (incomplete); rename it aside and rerun')
|
| 175 |
+
for rel, digest in SOURCE_SHA256.items():
|
| 176 |
+
if sha(SRC_CODE / rel) != digest:
|
| 177 |
+
raise SystemExit(f'Source code differs from the reviewed copy: {SRC_CODE / rel}')
|
| 178 |
+
if sha(AUG_SRC) != AUG_SHA256:
|
| 179 |
+
raise SystemExit(f'{AUG_SRC} differs from the reviewed hb_aug_v2 ({AUG_SHA256[:12]})')
|
| 180 |
+
if sha(REF_DELTA) != REF_DELTA_SHA256:
|
| 181 |
+
raise SystemExit(f'{REF_DELTA} differs from the reviewed state44 reference ({REF_DELTA_SHA256[:12]})')
|
| 182 |
+
ROOT.mkdir(parents=True, exist_ok=True)
|
| 183 |
+
tmp = ROOT / f'.HumanWAM.tmp{os.getpid()}'
|
| 184 |
+
shutil.copytree(SRC_CODE, tmp, symlinks=True, ignore=shutil.ignore_patterns('__pycache__', '*.pyc'))
|
| 185 |
+
loader = tmp / 'src/imagewam/datasets/wholebody.py'
|
| 186 |
+
text = loader.read_text(encoding='utf-8')
|
| 187 |
+
for old, new, count in LOADER_PATCHES:
|
| 188 |
+
if text.count(old) != count:
|
| 189 |
+
raise SystemExit(f'Loader patch anchor found {text.count(old)}x (want {count}): {old[:60]!r}')
|
| 190 |
+
text = text.replace(old, new)
|
| 191 |
+
if '149' in text:
|
| 192 |
+
raise SystemExit('Loader still mentions 149 after patching')
|
| 193 |
+
loader.write_text(text, encoding='utf-8')
|
| 194 |
+
added = {'src/imagewam/datasets/hb_aug_v2.py': AUG_SRC,
|
| 195 |
+
'src/imagewam/datasets/wholebody_hbaug.py': H / 'wholebody_hbaug.py',
|
| 196 |
+
'scripts/wholebody/train_deltaflow.py': H / 'train_deltaflow.py',
|
| 197 |
+
'src/imagewam/wholebody_delta.py': REF_DELTA}
|
| 198 |
+
for rel, source in added.items():
|
| 199 |
+
shutil.copyfile(source, tmp / rel)
|
| 200 |
+
files = ['src/imagewam/datasets/wholebody.py', *added, *[k for k in SOURCE_SHA256 if k != 'src/imagewam/datasets/wholebody.py']]
|
| 201 |
+
dump(tmp / 'deltaflow-code.json', dict(
|
| 202 |
+
source=str(SRC_CODE), hb_aug_v2_source=str(AUG_SRC), created=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
| 203 |
+
changes=['wholebody.py: train exactly episodes 0-99 ("first_100"), later episodes shelved',
|
| 204 |
+
'hb_aug_v2.py: unchanged copy of the shared spec-v2 implementation',
|
| 205 |
+
'wholebody_hbaug.py: WholeBodyHBAugV2LeRobotDataset(WholeBodyRobustLeRobotDataset), visual aug -> '
|
| 206 |
+
'hb_aug_v2, 44-D proprio, NO joint velocity (wholebody_velocity.py stays in the tree, unused); '
|
| 207 |
+
'proprio aligned with GR00T A20 (per-sample perturb_joints, no delay, no history drop; 2026-11-12)',
|
| 208 |
+
'train_deltaflow.py: 40-epoch entry, full state at every save, --resume',
|
| 209 |
+
'wholebody_delta.py: the state44 copy of real30-from-sim60-20261108 (the B6 60-epoch copy hard-codes '
|
| 210 |
+
'the 73-D velocity state)'],
|
| 211 |
+
sha256={rel: sha(tmp / rel) for rel in files}))
|
| 212 |
+
tmp.rename(CODE)
|
| 213 |
+
log('code copy built:', CODE)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
PROPRIO_DIM = 44 # body 29 + hands 12 + gravity 3 (NO velocity)
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def no_velocity(cfg, where):
|
| 220 |
+
"""Reject any joint-velocity remnant in a training config (user rule 2026-11-10)."""
|
| 221 |
+
def walk(value, path):
|
| 222 |
+
if isinstance(value, dict):
|
| 223 |
+
for key, item in value.items():
|
| 224 |
+
if 'velocity' in str(key).lower() and item not in ([], None, False, 0):
|
| 225 |
+
raise SystemExit(f'{where}: velocity setting left in the config: {path}.{key}={item!r}')
|
| 226 |
+
walk(item, f'{path}.{key}')
|
| 227 |
+
elif isinstance(value, list):
|
| 228 |
+
for i, item in enumerate(value):
|
| 229 |
+
walk(item, f'{path}[{i}]')
|
| 230 |
+
elif isinstance(value, str) and 'velocity' in value.lower() and 'no velocity' not in value.lower():
|
| 231 |
+
raise SystemExit(f'{where}: velocity reference left in the config: {path}={value!r}')
|
| 232 |
+
walk(cfg, 'cfg')
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def prepare_task(task, wb):
|
| 236 |
+
spec = TASKS[task]
|
| 237 |
+
R = ROOT / task
|
| 238 |
+
if (R / 'prepared.ok').is_file():
|
| 239 |
+
log(task, 'already prepared, skipping')
|
| 240 |
+
return
|
| 241 |
+
if R.exists():
|
| 242 |
+
aside = ROOT / f'{task}.incomplete-{time.strftime("%Y%m%dT%H%M%S")}'
|
| 243 |
+
R.rename(aside)
|
| 244 |
+
log(task, 'half-prepared directory moved to', aside)
|
| 245 |
+
D, old = spec['data'], spec['old']
|
| 246 |
+
(R / 'run').mkdir(parents=True)
|
| 247 |
+
log(task, 'dataset', D)
|
| 248 |
+
|
| 249 |
+
meta = wb.load_metadata(D)
|
| 250 |
+
ids = [int(e['episode_index']) for e in meta['episodes']]
|
| 251 |
+
lengths = {int(e['episode_index']): int(e['length']) for e in meta['episodes']}
|
| 252 |
+
train = list(range(EPISODES))
|
| 253 |
+
if ((spec['total_episodes'] is not None and len(ids) != spec['total_episodes']) or len(ids) < EPISODES
|
| 254 |
+
or ids[:EPISODES] != train):
|
| 255 |
+
raise SystemExit(f'{task}: expected {spec["total_episodes"]} episodes 0..N-1, found {len(ids)}')
|
| 256 |
+
shelved = ids[EPISODES:]
|
| 257 |
+
if old is None:
|
| 258 |
+
old_manifest = new_manifest(task, spec, meta, ids)
|
| 259 |
+
else:
|
| 260 |
+
old_manifest = json.loads((old / 'split-manifest.json').read_text())
|
| 261 |
+
if sorted(old_manifest['train_episode_ids'] + old_manifest['validation_episode_ids']) != ids:
|
| 262 |
+
raise SystemExit(f'{task}: previous split manifest does not cover this dataset')
|
| 263 |
+
basis = ('Fixed first 100 episodes (episode_index 0-99) per project rule 2026-11-09; '
|
| 264 |
+
+ (f'episodes {shelved[0]}-{shelved[-1]} are shelved (listed as validation ids, never trained, '
|
| 265 |
+
'no validation run)' if shelved else 'the collection has exactly 100 episodes'))
|
| 266 |
+
manifest = dict(old_manifest, train_episode_ids=train, validation_episode_ids=shelved, shelved_episode_ids=shelved,
|
| 267 |
+
split_basis=basis,
|
| 268 |
+
previous_manifest=(str(old / 'split-manifest.json') if old is not None else 'none (generated)'))
|
| 269 |
+
dump(R / 'split-manifest.json', manifest)
|
| 270 |
+
wb.apply_split(meta, str(R / 'split-manifest.json')) # provenance hashes, joint names, groups
|
| 271 |
+
|
| 272 |
+
log(task, 'fitting normalization statistics on episodes 0-99')
|
| 273 |
+
wb.prepare_statistics(str(D), str(R / 'stats'), train_split_mode='first_100',
|
| 274 |
+
split_manifest_path=str(R / 'split-manifest.json'))
|
| 275 |
+
stats = json.loads((R / 'stats/dataset_stats.json').read_text())
|
| 276 |
+
frames = sum(lengths[i] for i in train)
|
| 277 |
+
if (stats['train_episode_ids'] != train or stats['heldout_episode_ids'] != shelved
|
| 278 |
+
or stats['train_frames'] != frames or stats['schema'] != wb.STATS_SCHEMA):
|
| 279 |
+
raise SystemExit(f'{task}: statistics do not cover exactly episodes 0-99')
|
| 280 |
+
|
| 281 |
+
u = math.ceil(frames / (WORLD * MICRO))
|
| 282 |
+
if u != math.ceil(math.ceil(frames / MICRO) / WORLD):
|
| 283 |
+
raise SystemExit('updates_per_epoch mismatch')
|
| 284 |
+
total, save_steps, warmup, n_epochs = fixed_schedule(u)
|
| 285 |
+
|
| 286 |
+
prompt = wb.DEFAULT_PROMPT.format(task=meta['tasks'][0]['task'])
|
| 287 |
+
qinfo = None
|
| 288 |
+
if old is not None:
|
| 289 |
+
shutil.copytree(old / 'qwen-text', R / 'qwen-text')
|
| 290 |
+
else:
|
| 291 |
+
import qwen_cache
|
| 292 |
+
ref_prompt = wb.DEFAULT_PROMPT.format(task=wb.load_metadata(TASKS['a3']['data'])['tasks'][0]['task'])
|
| 293 |
+
_, qinfo = qwen_cache.make(prompt, R / 'qwen-text', CODE, QWEN3, FLUX2_SRC, ref_prompt, TEMPLATE / 'qwen-text')
|
| 294 |
+
log(task, 'qwen cache computed', qinfo)
|
| 295 |
+
cache = R / 'qwen-text' / f'{hashlib.sha256(prompt.encode()).hexdigest()}.qwen3_flux2_len128.pt'
|
| 296 |
+
if not cache.is_file():
|
| 297 |
+
raise SystemExit(f'{task}: instruction cache missing in copied qwen-text: {cache.name}')
|
| 298 |
+
|
| 299 |
+
source_yaml = (old if old is not None else TEMPLATE) / 'run/production.yaml'
|
| 300 |
+
cfg = yaml.safe_load(source_yaml.read_text())
|
| 301 |
+
cfg.update(output_dir=str(R / 'run/production'), num_epochs=n_epochs, max_steps=total,
|
| 302 |
+
lr_scheduler_total_steps=total, warmup_steps=warmup, save_every=0, save_steps=save_steps,
|
| 303 |
+
save_at_end=True, keep_latest_weights=KEEP_CKPTS, keep_latest_state_only=True, resume=None, eval_every=0)
|
| 304 |
+
if cfg['batch_size'] != MICRO or cfg['gradient_accumulation_steps'] != 1 or cfg['data']['val'] is not None:
|
| 305 |
+
raise SystemExit(f'{task}: unexpected batch / accumulation / validation in {source_yaml}')
|
| 306 |
+
cfg['wandb'].update(enabled=False, mode='disabled', name=f'deltaflow-{task}-40ep-augv2-aligned')
|
| 307 |
+
d = cfg['data']['train']
|
| 308 |
+
d.update(_target_=DATASET_TARGET, dataset_dir=str(D), stats_path=str(R / 'stats/dataset_stats.json'),
|
| 309 |
+
qwen_text_cache_dir=str(R / 'qwen-text'), split_manifest_path=str(R / 'split-manifest.json'),
|
| 310 |
+
train_split_mode='first_100')
|
| 311 |
+
for key in [k for k in d if 'velocity' in k]: # the previous run's 73-D loader keys (velocity_cache, noise)
|
| 312 |
+
del d[key]
|
| 313 |
+
d['image_augmentation'] = augmentation_config()
|
| 314 |
+
pa = d['proprio_augmentation']
|
| 315 |
+
pa.update(control_hz=spec['control_hz'], recording_hz=30, velocity_channels=[])
|
| 316 |
+
for key, value in dict(enabled=True, joint_bias_degrees=2.0, joint_noise_std_degrees=0.25, delay_probability=0.15,
|
| 317 |
+
max_delay_control_cycles=2, history_drop_probability=0.15,
|
| 318 |
+
affected_angle_channels=list(range(29)), bias_scope='fixed_per_episode').items():
|
| 319 |
+
if pa.get(key) != value:
|
| 320 |
+
raise SystemExit(f'{task}: proprio_augmentation.{key}={pa.get(key)!r} differs from spec v2 ({value!r})')
|
| 321 |
+
pa.update(ALIGNED_PROPRIO)
|
| 322 |
+
m = cfg['model']
|
| 323 |
+
if (m['proprio_dim'], m['action_state_dim'], m['expected_action_dim'], m['expected_action_horizon']) not in (
|
| 324 |
+
(73, 73, 76, 16), (44, 44, 76, 16)):
|
| 325 |
+
raise SystemExit(f'{task}: source model is not the 76-D action / horizon-16 Delta-flow')
|
| 326 |
+
m.update(proprio_dim=PROPRIO_DIM, action_state_dim=PROPRIO_DIM) # = reference real30-from-sim60 (no velocity)
|
| 327 |
+
c = cfg['wholebody_contract']
|
| 328 |
+
for key in [k for k in c if 'velocity' in k]:
|
| 329 |
+
del c[key]
|
| 330 |
+
c.update(stats_schema=wb.STATS_SCHEMA, statistics_sha256=sha(R / 'stats/dataset_stats.json'),
|
| 331 |
+
smoke_output_dir=None, production_output_dir=str(R / 'run/production'), updates_per_epoch=u,
|
| 332 |
+
augmentation=AUG_VERSION, proprio_augmentation='aligned with GR00T A20 (per-sample perturb_joints, no delay, no history drop)', proprio_layout='body29 + hands12 + gravity3 = 44 (no velocity)',
|
| 333 |
+
train_episode_ids='0-99', shelved_episode_ids=(f'{shelved[0]}-{shelved[-1]}' if shelved else 'none'))
|
| 334 |
+
c['measured_loader'].update(dataset_frames=frames, raw_loader_batches=math.ceil(frames / MICRO),
|
| 335 |
+
per_rank_microbatches=u, world_size=WORLD, microbatch=MICRO)
|
| 336 |
+
old = old if old is not None else TEMPLATE
|
| 337 |
+
old_prefix = str(old) + '/'
|
| 338 |
+
|
| 339 |
+
def walk(value):
|
| 340 |
+
if isinstance(value, dict):
|
| 341 |
+
for item in value.values():
|
| 342 |
+
walk(item)
|
| 343 |
+
elif isinstance(value, list):
|
| 344 |
+
for item in value:
|
| 345 |
+
walk(item)
|
| 346 |
+
elif isinstance(value, str) and (value.startswith(old_prefix) or value == str(old)):
|
| 347 |
+
raise SystemExit(f'{task}: config still points into the old experiment: {value}')
|
| 348 |
+
walk(cfg)
|
| 349 |
+
no_velocity(cfg, task)
|
| 350 |
+
c['source_config'] = str(source_yaml)
|
| 351 |
+
(R / 'run/production.yaml').write_text(yaml.safe_dump(cfg, sort_keys=False), encoding='utf-8')
|
| 352 |
+
|
| 353 |
+
dump(R / 'data-verified.json', dict(
|
| 354 |
+
task=task, dataset_dir=str(D), total_episodes_in_collection=len(ids), episodes=EPISODES,
|
| 355 |
+
train_episode_ids=train, shelved_episode_ids=shelved, frames=frames, updates_per_epoch=u, epochs=None, fixed_steps=True,
|
| 356 |
+
epochs_equivalent=round(total / u, 2),
|
| 357 |
+
max_steps=total, warmup_steps=warmup, save_steps=save_steps, control_hz=spec['control_hz'],
|
| 358 |
+
recording_hz=30, statistics_sha256=c['statistics_sha256'], proprio_dim=PROPRIO_DIM, joint_velocity=False,
|
| 359 |
+
augmentation=AUG_VERSION, proprio_augmentation=ALIGNED_PROPRIO, dataset_source=spec.get('source'),
|
| 360 |
+
qwen_cache=(qinfo or 'copied from the previous run'),
|
| 361 |
+
code=str(CODE), code_record_sha256=sha(CODE / 'deltaflow-code.json')))
|
| 362 |
+
|
| 363 |
+
log(task, 'loading two real augmented samples (CPU)')
|
| 364 |
+
from hydra.utils import instantiate
|
| 365 |
+
from omegaconf import OmegaConf
|
| 366 |
+
dataset = instantiate(OmegaConf.load(R / 'run/production.yaml').data.train)
|
| 367 |
+
if len(dataset) != frames or list(dataset.episode_ids) != train:
|
| 368 |
+
raise SystemExit(f'{task}: dataset length {len(dataset)} / episodes differ from {frames} / 0-99')
|
| 369 |
+
import torch
|
| 370 |
+
first, sample = dataset[0], dataset[min(20, len(dataset) - 1)]
|
| 371 |
+
if (tuple(first['video'].shape) != (3, 2, 224, 448) or tuple(first['proprio'].shape) != (16, PROPRIO_DIM)
|
| 372 |
+
or tuple(sample['proprio'].shape) != (16, PROPRIO_DIM) or tuple(sample['action'].shape) != (16, 76)
|
| 373 |
+
or not torch.isfinite(sample['proprio']).all() or dataset.last_augmentation['visual']['boost'] is None):
|
| 374 |
+
raise SystemExit(f'{task}: sample schema check failed')
|
| 375 |
+
perturb = []
|
| 376 |
+
for i in range(0, frames, max(1, frames // 40)):
|
| 377 |
+
dataset[i]
|
| 378 |
+
num = dataset.last_augmentation['numeric']
|
| 379 |
+
if num['delay_recorded_frames'] != 0 or num['history_drop_rows']:
|
| 380 |
+
raise SystemExit(f'{task}: delay / history drop still active: {num}')
|
| 381 |
+
perturb.append(num['joint_perturbation_degrees'])
|
| 382 |
+
perturb = np.asarray(perturb)
|
| 383 |
+
if perturb.shape[1] != 29 or np.abs(perturb).max() > 2.0 + 0.75 + 1e-3 or perturb.std(axis=0).min() < 0.5:
|
| 384 |
+
raise SystemExit(f'{task}: joint perturbation is not per-sample +-2 deg + 0.25 deg noise')
|
| 385 |
+
dump(R / 'prepared.ok', dict(frames=frames, updates_per_epoch=u, max_steps=total, save_steps=save_steps,
|
| 386 |
+
proprio_check=dict(samples=len(perturb), max_abs_deg=float(np.abs(perturb).max()),
|
| 387 |
+
std_deg=float(perturb.std()))))
|
| 388 |
+
log(task, 'PREPARED', json.dumps(dict(frames=frames, updates_per_epoch=u, max_steps=total, save_steps=save_steps,
|
| 389 |
+
warmup=warmup, control_hz=spec['control_hz'])))
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def main():
|
| 393 |
+
tasks = sys.argv[1:] or [*TASKS, *H2_TASKS]
|
| 394 |
+
for task in tasks:
|
| 395 |
+
if task not in TASKS and task not in H2_TASKS:
|
| 396 |
+
raise SystemExit(f'unknown task {task}; choose from {[*TASKS, *H2_TASKS]}')
|
| 397 |
+
g1 = [task for task in tasks if task in TASKS]
|
| 398 |
+
if g1:
|
| 399 |
+
build_code()
|
| 400 |
+
sys.path.insert(0, str(CODE / 'src'))
|
| 401 |
+
os.environ.setdefault('HF_HUB_OFFLINE', '1')
|
| 402 |
+
import imagewam.datasets.wholebody as wb
|
| 403 |
+
if not wb.__file__.startswith(str(CODE / 'src')):
|
| 404 |
+
raise SystemExit(f'imported the wrong imagewam: {wb.__file__}')
|
| 405 |
+
for task in g1:
|
| 406 |
+
prepare_task(task, wb)
|
| 407 |
+
h2 = [task for task in tasks if task in H2_TASKS]
|
| 408 |
+
if h2:
|
| 409 |
+
# The H2 branch ships its own imagewam package: prepare it in a separate interpreter.
|
| 410 |
+
import subprocess
|
| 411 |
+
rc = subprocess.call([sys.executable, str(H / 'prepare_deltaflow_h2.py'), *h2])
|
| 412 |
+
if rc != 0:
|
| 413 |
+
raise SystemExit(f'H2 preparation failed (rc {rc})')
|
| 414 |
+
log('PREPARE_DONE', ' '.join(tasks))
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
if __name__ == '__main__':
|
| 418 |
+
main()
|
code/hb_deltaflow/prepare_deltaflow_a18.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A18 (H2 pour water) preparation for the Delta-flow 40-epoch runs. CPU only, node 0, idempotent.
|
| 2 |
+
|
| 3 |
+
$P/venv/bin/python $G/hb_deltaflow/prepare_deltaflow_a18.py (also called by prepare_deltaflow.py a18)
|
| 4 |
+
NO joint velocity (user rule 2026-11-10): the model is the original 78-D H2 model of a18-h2-pour-from-scratch-20261108
|
| 5 |
+
(imagewam/h2_pour_delta.py, unchanged; same as HF g1-basket-transfer-imagewam-deltaflow76 h2-a18-pour-water/epoch-040,
|
| 6 |
+
proprio_dim 78 = 75 joints + gravity 3). No velocity cache or statistics are produced.
|
| 7 |
+
Creates:
|
| 8 |
+
$G/deltaflow-runs/HumanWAM-h2/ copy of $P/a18-h2-pour-from-scratch-20261108/HumanWAM (hash-checked) plus
|
| 9 |
+
src/imagewam/datasets/hb_aug_v2.py unchanged copy of $G/hb_groot/hb_aug_v2.py
|
| 10 |
+
src/imagewam/datasets/h2_hbaug.py H2HBAugV2Dataset (aug v2 + 31-joint proprio aug, 78-D, no velocity)
|
| 11 |
+
scripts/wholebody/train_deltaflow.py the shared 40-epoch entry
|
| 12 |
+
$G/deltaflow-runs/a18/{stats/normalization.json, qwen-text/, run/production.yaml, data-verified.json, prepared.ok}
|
| 13 |
+
"""
|
| 14 |
+
import hashlib
|
| 15 |
+
import inspect
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
import os
|
| 19 |
+
import shutil
|
| 20 |
+
import sys
|
| 21 |
+
import time
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
import yaml
|
| 25 |
+
|
| 26 |
+
import prepare_deltaflow as base
|
| 27 |
+
|
| 28 |
+
H, G, P, ROOT = base.H, base.G, base.P, base.ROOT
|
| 29 |
+
OLD = P / 'a18-h2-pour-from-scratch-20261108'
|
| 30 |
+
SRC_CODE = OLD / 'HumanWAM'
|
| 31 |
+
DATA = P / 'a18-h2-dataset-audit-8e956a10' / 'dataset'
|
| 32 |
+
CODE = ROOT / 'HumanWAM-h2'
|
| 33 |
+
R = ROOT / 'a18'
|
| 34 |
+
EPISODES, FRAMES, BODY, STATE, ACTION = 100, 72752, 31, 78, 75
|
| 35 |
+
DATASET_TARGET = 'imagewam.datasets.h2_hbaug.H2HBAugV2Dataset'
|
| 36 |
+
SOURCE_SHA256 = { # = h2-run-manifest.json file_sha256 / reviewed local copy
|
| 37 |
+
'src/imagewam/datasets/h2_pour.py': 'd61ec58b83c20621488899c023a348fed52a7881f8faa4f6ba277fcb9c399168',
|
| 38 |
+
'src/imagewam/h2_pour_delta.py': '69bf26169559b082a0a45152512497998163b63c69723d1af606dcf82eca47fc',
|
| 39 |
+
'src/imagewam/datasets/wholebody.py': '4d81de5a73109752128a41a65b01121afe12de15cac42f215157c4e30008ccf9',
|
| 40 |
+
'src/imagewam/trainer.py': 'cdcd48e09d1d7b81cae3a829bd415ed4b53ff1b290aacac1ce2217876f475b1a',
|
| 41 |
+
'src/imagewam/runtime.py': '3ebc45f61b0c22fb659012a7d76bea671a42e1e528b5c917ae1e8827e88b890b',
|
| 42 |
+
'scripts/wholebody/train_h2_pour.py': '69c6974418b9ed02cae4c9cca7177e3c32a213d9cc833ecd3d0c95489d901a5e',
|
| 43 |
+
'scripts/accelerate_configs/accelerate_zero1_ds.yaml': '5b9e24f38c199fdff6bbd17981d86e2624f855f707660094c99f711efc8e0833',
|
| 44 |
+
}
|
| 45 |
+
sha, dump, log = base.sha, base.dump, base.log
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def build_code():
|
| 49 |
+
record = CODE / 'deltaflow-code.json'
|
| 50 |
+
if record.is_file():
|
| 51 |
+
for rel, digest in json.loads(record.read_text())['sha256'].items():
|
| 52 |
+
if sha(CODE / rel) != digest:
|
| 53 |
+
raise SystemExit(f'{CODE / rel} changed after preparation; rename {CODE} aside and rerun')
|
| 54 |
+
log('H2 code copy present and unchanged:', CODE)
|
| 55 |
+
return
|
| 56 |
+
if CODE.exists():
|
| 57 |
+
raise SystemExit(f'{CODE} exists without deltaflow-code.json (incomplete); rename it aside and rerun')
|
| 58 |
+
for rel, digest in SOURCE_SHA256.items():
|
| 59 |
+
if sha(SRC_CODE / rel) != digest:
|
| 60 |
+
raise SystemExit(f'H2 source differs from the reviewed copy: {SRC_CODE / rel}')
|
| 61 |
+
if sha(base.AUG_SRC) != base.AUG_SHA256:
|
| 62 |
+
raise SystemExit(f'{base.AUG_SRC} differs from the reviewed hb_aug_v2')
|
| 63 |
+
ROOT.mkdir(parents=True, exist_ok=True)
|
| 64 |
+
tmp = ROOT / f'.HumanWAM-h2.tmp{os.getpid()}'
|
| 65 |
+
shutil.copytree(SRC_CODE, tmp, symlinks=True, ignore=shutil.ignore_patterns('__pycache__', '*.pyc'))
|
| 66 |
+
added = {'src/imagewam/datasets/hb_aug_v2.py': base.AUG_SRC,
|
| 67 |
+
'src/imagewam/datasets/h2_hbaug.py': H / 'h2_hbaug.py',
|
| 68 |
+
'scripts/wholebody/train_deltaflow.py': H / 'train_deltaflow.py'}
|
| 69 |
+
for rel, source in added.items():
|
| 70 |
+
shutil.copyfile(source, tmp / rel)
|
| 71 |
+
files = [*added, *SOURCE_SHA256]
|
| 72 |
+
dump(tmp / 'deltaflow-code.json', dict(
|
| 73 |
+
source=str(SRC_CODE), hb_aug_v2_source=str(base.AUG_SRC),
|
| 74 |
+
created=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
| 75 |
+
changes=['h2_pour_delta.py: unchanged original 78-D H2 model (no joint velocity)',
|
| 76 |
+
'hb_aug_v2.py: unchanged copy of the shared spec-v2 implementation',
|
| 77 |
+
'h2_hbaug.py: H2HBAugV2Dataset (aug v2 + 31-joint proprio aug, 78-D state, no velocity)',
|
| 78 |
+
'train_deltaflow.py: shared 40-epoch entry, full state at every save, --resume'],
|
| 79 |
+
sha256={rel: sha(tmp / rel) for rel in files}))
|
| 80 |
+
tmp.rename(CODE)
|
| 81 |
+
log('H2 code copy built:', CODE)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def prepare():
|
| 85 |
+
if (R / 'prepared.ok').is_file():
|
| 86 |
+
log('a18 already prepared, skipping')
|
| 87 |
+
return
|
| 88 |
+
if R.exists():
|
| 89 |
+
aside = ROOT / f'a18.incomplete-{time.strftime("%Y%m%dT%H%M%S")}'
|
| 90 |
+
R.rename(aside)
|
| 91 |
+
log('a18 half-prepared directory moved to', aside)
|
| 92 |
+
(R / 'run').mkdir(parents=True)
|
| 93 |
+
(R / 'stats').mkdir()
|
| 94 |
+
sys.path.insert(0, str(CODE / 'src'))
|
| 95 |
+
os.environ.setdefault('HF_HUB_OFFLINE', '1')
|
| 96 |
+
from imagewam.datasets import h2_pour
|
| 97 |
+
if not h2_pour.__file__.startswith(str(CODE / 'src')):
|
| 98 |
+
raise SystemExit(f'imported the wrong imagewam: {h2_pour.__file__}')
|
| 99 |
+
if h2_pour.STATE_DIM != STATE or h2_pour.ACTION_DIM != ACTION:
|
| 100 |
+
raise SystemExit(f'h2_pour dims {h2_pour.STATE_DIM}/{h2_pour.ACTION_DIM}, expected {STATE}/{ACTION}')
|
| 101 |
+
episodes, instruction, names = h2_pour.load_collection(DATA) # exactly 100 episodes / 72752 frames
|
| 102 |
+
lengths = [int(e['length']) for e in episodes]
|
| 103 |
+
if [e['episode_index'] for e in episodes] != list(range(EPISODES)) or sum(lengths) != FRAMES:
|
| 104 |
+
raise SystemExit('A18 collection is not episodes 0-99 / 72752 frames')
|
| 105 |
+
if list(h2_pour.CAMERAS) != ['observation.images.head', 'observation.images.chest']:
|
| 106 |
+
raise SystemExit(f'Unexpected H2 cameras {h2_pour.CAMERAS}')
|
| 107 |
+
|
| 108 |
+
log('a18 normalization (q01/q99 of state78 / action75 over episodes 0-99)')
|
| 109 |
+
h2_pour.fit(DATA, R / 'stats/normalization.json')
|
| 110 |
+
|
| 111 |
+
u = math.ceil(FRAMES / (base.WORLD * base.MICRO)) # 285
|
| 112 |
+
total = base.EPOCHS * u
|
| 113 |
+
save_steps = [e * u for e in range(base.SAVE_EVERY_EPOCHS, base.EPOCHS + 1, base.SAVE_EVERY_EPOCHS)]
|
| 114 |
+
shutil.copytree(OLD / 'qwen-text', R / 'qwen-text')
|
| 115 |
+
prompt = h2_pour.DEFAULT_PROMPT.format(task=instruction)
|
| 116 |
+
cache = R / 'qwen-text' / f'{hashlib.sha256(prompt.encode()).hexdigest()}.qwen3_flux2_len128.pt'
|
| 117 |
+
if not cache.is_file():
|
| 118 |
+
raise SystemExit(f'a18: instruction cache missing in copied qwen-text: {cache.name}')
|
| 119 |
+
|
| 120 |
+
source_yaml = OLD / 'run/production.yaml'
|
| 121 |
+
cfg = yaml.safe_load(source_yaml.read_text())
|
| 122 |
+
cfg.update(output_dir=str(R / 'run/production'), num_epochs=base.EPOCHS, max_steps=total,
|
| 123 |
+
lr_scheduler_total_steps=total, warmup_steps=base.WARMUP_EPOCHS * u, save_every=0,
|
| 124 |
+
save_steps=save_steps, save_at_end=True, keep_latest_weights=4, keep_latest_state_only=True,
|
| 125 |
+
resume=None, eval_every=0)
|
| 126 |
+
if cfg['batch_size'] != base.MICRO or cfg['gradient_accumulation_steps'] != 1 or cfg['data']['val'] is not None:
|
| 127 |
+
raise SystemExit('a18: unexpected batch / accumulation / validation in the H2 config')
|
| 128 |
+
cfg['wandb'].update(enabled=False, mode='disabled', name='deltaflow-a18-40ep-augv2', group='h2-action75-state78')
|
| 129 |
+
old_train = cfg['data']['train']
|
| 130 |
+
cfg['data']['train'] = dict(
|
| 131 |
+
_target_=DATASET_TARGET, dataset_dir=str(DATA), stats_path=str(R / 'stats/normalization.json'),
|
| 132 |
+
qwen_text_cache_dir=str(R / 'qwen-text'), qwen_context_len=old_train['qwen_context_len'],
|
| 133 |
+
qwen_text_dim=old_train['qwen_text_dim'], video_cache_frames=64,
|
| 134 |
+
image_augmentation=base.augmentation_config(),
|
| 135 |
+
proprio_augmentation=dict(
|
| 136 |
+
enabled=True, seed=20261106, joint_bias_degrees=2.0, joint_noise_std_degrees=0.25, delay_probability=0.15,
|
| 137 |
+
max_delay_control_cycles=2, control_hz=30, recording_hz=30, history_drop_probability=0.15,
|
| 138 |
+
affected_angle_channels=list(range(BODY)), velocity_channels=[],
|
| 139 |
+
action_label_noise=False, bias_scope='fixed_per_episode', body_joints='observation.state[0:31] incl. head',
|
| 140 |
+
start_pose_library='not_applied_offline', render_randomization='not_applied_offline'))
|
| 141 |
+
m = cfg['model']
|
| 142 |
+
if (m['expected_action_dim'], m['action_dit_config']['action_dim'], len(m['loss']['action_a2a']['source_noise_dim_mask']),
|
| 143 |
+
m['proprio_dim'], m['action_state_dim'], m['expected_action_horizon']) != (ACTION, ACTION, ACTION, STATE, STATE, 16):
|
| 144 |
+
raise SystemExit('a18: the H2 config is not the 78-D state / 75-D action Delta-flow it should be')
|
| 145 |
+
if 'h2_pour_delta' not in str(m.get('_target_', '')):
|
| 146 |
+
raise SystemExit(f"a18: unexpected model target {m.get('_target_')!r} (want the original h2_pour_delta)")
|
| 147 |
+
cfg['wholebody_contract'] = dict(
|
| 148 |
+
task='a18', robot='h2_sharpa', world_size=base.WORLD, zero_stage=1, fresh_base=True,
|
| 149 |
+
production_output_dir=str(R / 'run/production'), updates_per_epoch=u, train_episode_ids='0-99',
|
| 150 |
+
statistics_sha256=sha(R / 'stats/normalization.json'), augmentation=base.AUG_VERSION,
|
| 151 |
+
state_layout='joints75 + gravity3 = 78 (no velocity)',
|
| 152 |
+
action_layout='body_target31 + hands44 = 75', cameras=list(h2_pour.CAMERAS),
|
| 153 |
+
measured_loader=dict(dataset_frames=FRAMES, raw_loader_batches=math.ceil(FRAMES / base.MICRO),
|
| 154 |
+
per_rank_microbatches=u, world_size=base.WORLD, microbatch=base.MICRO,
|
| 155 |
+
gradient_accumulation=1),
|
| 156 |
+
source_config=str(source_yaml))
|
| 157 |
+
for value in (cfg['output_dir'], cfg['data']['train']['stats_path'], cfg['data']['train']['qwen_text_cache_dir']):
|
| 158 |
+
if value.startswith(str(OLD) + '/'):
|
| 159 |
+
raise SystemExit(f'a18: config still points into the old experiment: {value}')
|
| 160 |
+
base.no_velocity(cfg, 'a18')
|
| 161 |
+
(R / 'run/production.yaml').write_text(yaml.safe_dump(cfg, sort_keys=False), encoding='utf-8')
|
| 162 |
+
previous = OLD / 'normalization.json'
|
| 163 |
+
dump(R / 'data-verified.json', dict(
|
| 164 |
+
task='a18', dataset_dir=str(DATA), total_episodes_in_collection=EPISODES, episodes=EPISODES,
|
| 165 |
+
train_episode_ids=list(range(EPISODES)), shelved_episode_ids=[], frames=FRAMES, updates_per_epoch=u,
|
| 166 |
+
epochs=base.EPOCHS, max_steps=total, warmup_steps=base.WARMUP_EPOCHS * u, save_steps=save_steps,
|
| 167 |
+
control_hz=30, recording_hz=30, statistics_sha256=sha(R / 'stats/normalization.json'),
|
| 168 |
+
statistics_equal_previous_a18_run=previous.is_file() and sha(previous) == sha(R / 'stats/normalization.json'),
|
| 169 |
+
augmentation=base.AUG_VERSION, state_dim=STATE, action_dim=ACTION, joint_velocity=False,
|
| 170 |
+
code=str(CODE), code_record_sha256=sha(CODE / 'deltaflow-code.json')))
|
| 171 |
+
|
| 172 |
+
log('a18 model module check (no weights loaded)')
|
| 173 |
+
import imagewam.h2_pour_delta as model_module
|
| 174 |
+
factory = inspect.signature(model_module.create_imagewam_flux2_wholebody_delta)
|
| 175 |
+
if (factory.parameters['action_state_dim'].default != STATE
|
| 176 |
+
or factory.parameters['expected_action_dim'].default != ACTION):
|
| 177 |
+
raise SystemExit('a18: original H2 model module is not 78-D state / 75-D action')
|
| 178 |
+
|
| 179 |
+
log('a18 loading two real augmented samples (CPU)')
|
| 180 |
+
import torch
|
| 181 |
+
from hydra.utils import instantiate
|
| 182 |
+
from omegaconf import OmegaConf
|
| 183 |
+
dataset = instantiate(OmegaConf.load(R / 'run/production.yaml').data.train)
|
| 184 |
+
if len(dataset) != FRAMES or list(dataset.episode_ids) != list(range(EPISODES)):
|
| 185 |
+
raise SystemExit(f'a18: dataset length {len(dataset)} differs from {FRAMES}')
|
| 186 |
+
first, sample = dataset[0], dataset[145]
|
| 187 |
+
if (tuple(first['video'].shape) != (3, 2, 224, 448) or tuple(first['proprio'].shape) != (16, STATE)
|
| 188 |
+
or tuple(sample['proprio'].shape) != (16, STATE) or tuple(sample['proprio_dim_is_pad'].shape) != (STATE,)
|
| 189 |
+
or tuple(sample['action'].shape) != (16, ACTION) or tuple(sample['past_action'].shape) != (16, ACTION)
|
| 190 |
+
or not torch.isfinite(sample['proprio']).all() or dataset.last_augmentation['visual']['boost'] is None):
|
| 191 |
+
raise SystemExit('a18: sample schema check failed')
|
| 192 |
+
dump(R / 'prepared.ok', dict(frames=FRAMES, updates_per_epoch=u, max_steps=total, save_steps=save_steps))
|
| 193 |
+
log('a18 PREPARED', json.dumps(dict(frames=FRAMES, updates_per_epoch=u, max_steps=total, save_steps=save_steps,
|
| 194 |
+
warmup=base.WARMUP_EPOCHS * u, state_dim=STATE, action_dim=ACTION)))
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
if __name__ == '__main__':
|
| 198 |
+
build_code()
|
| 199 |
+
prepare()
|
code/hb_deltaflow/prepare_deltaflow_h2.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""H2 (A18 pour water, A17 unscrew bottle) preparation for the Delta-flow 40-epoch runs. CPU only, node 0, idempotent.
|
| 2 |
+
|
| 3 |
+
$P/venv/bin/python $G/hb_deltaflow/prepare_deltaflow_h2.py <a18|a17> (also called by prepare_deltaflow.py)
|
| 4 |
+
Generalizes prepare_deltaflow_a18.py (2026-11-12): proprio aligned with GR00T's A20 recipe (per-sample
|
| 5 |
+
hb_aug_v2.perturb_joints on the 31 H2 body joints, no delay, no history drop) and A17 added.
|
| 6 |
+
NO joint velocity (user rule 2026-11-10): the model is the original 78-D H2 model of a18-h2-pour-from-scratch-20261108
|
| 7 |
+
(imagewam/h2_pour_delta.py, unchanged; proprio_dim 78 = 75 joints + gravity 3).
|
| 8 |
+
Creates:
|
| 9 |
+
$G/deltaflow-runs/HumanWAM-h2/ copy of $P/a18-h2-pour-from-scratch-20261108/HumanWAM (hash-checked) plus
|
| 10 |
+
src/imagewam/datasets/hb_aug_v2.py unchanged copy of $G/hb_groot/hb_aug_v2.py
|
| 11 |
+
src/imagewam/datasets/h2_hbaug.py H2HBAugV2Dataset (aug v2 + aligned 31-joint proprio aug, 78-D)
|
| 12 |
+
scripts/wholebody/train_deltaflow.py the shared 40-epoch entry
|
| 13 |
+
src/imagewam/datasets/h2_pour.py frame total read from meta/info.json instead of the A18 constant 72752
|
| 14 |
+
$G/deltaflow-runs/<task>/{stats/normalization.json, qwen-text/, run/production.yaml, data-verified.json, prepared.ok}
|
| 15 |
+
"""
|
| 16 |
+
import hashlib
|
| 17 |
+
import inspect
|
| 18 |
+
import json
|
| 19 |
+
import math
|
| 20 |
+
import os
|
| 21 |
+
import shutil
|
| 22 |
+
import sys
|
| 23 |
+
import time
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
import yaml
|
| 27 |
+
|
| 28 |
+
import prepare_deltaflow as base
|
| 29 |
+
|
| 30 |
+
H, G, P, ROOT = base.H, base.G, base.P, base.ROOT
|
| 31 |
+
OLD = P / 'a18-h2-pour-from-scratch-20261108'
|
| 32 |
+
SRC_CODE = OLD / 'HumanWAM'
|
| 33 |
+
CODE = ROOT / 'HumanWAM-h2'
|
| 34 |
+
TASKS = {'a18': dict(data=P / 'a18-h2-dataset-audit-8e956a10' / 'dataset', frames=72752, qwen='copy',
|
| 35 |
+
source='a18 H2 pour water (audited 8e956a10)'),
|
| 36 |
+
'a17': dict(data=P / 'a17-unscrew-bottle-v1-407e6e2' / 'data', frames=115333, qwen='compute',
|
| 37 |
+
source='a17-unscrew-bottle-expert-v1 @407e6e2')}
|
| 38 |
+
EPISODES, BODY, STATE, ACTION = 100, 31, 78, 75
|
| 39 |
+
DATASET_TARGET = 'imagewam.datasets.h2_hbaug.H2HBAugV2Dataset'
|
| 40 |
+
SOURCE_SHA256 = { # = h2-run-manifest.json file_sha256 / reviewed local copy
|
| 41 |
+
'src/imagewam/datasets/h2_pour.py': 'd61ec58b83c20621488899c023a348fed52a7881f8faa4f6ba277fcb9c399168',
|
| 42 |
+
'src/imagewam/h2_pour_delta.py': '69bf26169559b082a0a45152512497998163b63c69723d1af606dcf82eca47fc',
|
| 43 |
+
'src/imagewam/datasets/wholebody.py': '4d81de5a73109752128a41a65b01121afe12de15cac42f215157c4e30008ccf9',
|
| 44 |
+
'src/imagewam/trainer.py': 'cdcd48e09d1d7b81cae3a829bd415ed4b53ff1b290aacac1ce2217876f475b1a',
|
| 45 |
+
'src/imagewam/runtime.py': '3ebc45f61b0c22fb659012a7d76bea671a42e1e528b5c917ae1e8827e88b890b',
|
| 46 |
+
'scripts/wholebody/train_h2_pour.py': '69c6974418b9ed02cae4c9cca7177e3c32a213d9cc833ecd3d0c95489d901a5e',
|
| 47 |
+
'scripts/accelerate_configs/accelerate_zero1_ds.yaml': '5b9e24f38c199fdff6bbd17981d86e2624f855f707660094c99f711efc8e0833',
|
| 48 |
+
}
|
| 49 |
+
H2_POUR_PATCHES = [ # make the A18-specific frame total generic (A17 has 115333 frames); everything else unchanged
|
| 50 |
+
("sum(row['length'] for row in episodes)!=72752", "sum(row['length'] for row in episodes)!=int(info['total_frames'])"),
|
| 51 |
+
("frames=72752,", "frames=sum(int(row['length']) for row in episodes),"),
|
| 52 |
+
]
|
| 53 |
+
ALIGNED = dict(delay_probability=0.0, max_delay_control_cycles=0, history_drop_probability=0.0,
|
| 54 |
+
bias_scope='per_sample', implementation='hb_aug_v2.perturb_joints',
|
| 55 |
+
aligned_with='GR00T A20 basket-transfer recipe (hb_groot/hb_launch.py), user 2026-11-12')
|
| 56 |
+
sha, dump, log = base.sha, base.dump, base.log
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def build_code():
|
| 60 |
+
record = CODE / 'deltaflow-code.json'
|
| 61 |
+
if record.is_file():
|
| 62 |
+
for rel, digest in json.loads(record.read_text())['sha256'].items():
|
| 63 |
+
if sha(CODE / rel) != digest:
|
| 64 |
+
raise SystemExit(f'{CODE / rel} changed after preparation; rename {CODE} aside and rerun')
|
| 65 |
+
log('H2 code copy present and unchanged:', CODE)
|
| 66 |
+
return
|
| 67 |
+
if CODE.exists():
|
| 68 |
+
raise SystemExit(f'{CODE} exists without deltaflow-code.json (incomplete); rename it aside and rerun')
|
| 69 |
+
for rel, digest in SOURCE_SHA256.items():
|
| 70 |
+
if sha(SRC_CODE / rel) != digest:
|
| 71 |
+
raise SystemExit(f'H2 source differs from the reviewed copy: {SRC_CODE / rel}')
|
| 72 |
+
if sha(base.AUG_SRC) != base.AUG_SHA256:
|
| 73 |
+
raise SystemExit(f'{base.AUG_SRC} differs from the reviewed hb_aug_v2')
|
| 74 |
+
ROOT.mkdir(parents=True, exist_ok=True)
|
| 75 |
+
tmp = ROOT / f'.HumanWAM-h2.tmp{os.getpid()}'
|
| 76 |
+
shutil.copytree(SRC_CODE, tmp, symlinks=True, ignore=shutil.ignore_patterns('__pycache__', '*.pyc'))
|
| 77 |
+
added = {'src/imagewam/datasets/hb_aug_v2.py': base.AUG_SRC,
|
| 78 |
+
'src/imagewam/datasets/h2_hbaug.py': H / 'h2_hbaug.py',
|
| 79 |
+
'scripts/wholebody/train_deltaflow.py': H / 'train_deltaflow.py'}
|
| 80 |
+
for rel, source in added.items():
|
| 81 |
+
shutil.copyfile(source, tmp / rel)
|
| 82 |
+
pour = tmp / 'src/imagewam/datasets/h2_pour.py'
|
| 83 |
+
text = pour.read_text(encoding='utf-8')
|
| 84 |
+
for old, new in H2_POUR_PATCHES:
|
| 85 |
+
if text.count(old) != 1:
|
| 86 |
+
raise SystemExit(f'h2_pour patch anchor found {text.count(old)}x: {old!r}')
|
| 87 |
+
text = text.replace(old, new)
|
| 88 |
+
pour.write_text(text, encoding='utf-8')
|
| 89 |
+
files = [*added, *SOURCE_SHA256]
|
| 90 |
+
dump(tmp / 'deltaflow-code.json', dict(
|
| 91 |
+
source=str(SRC_CODE), hb_aug_v2_source=str(base.AUG_SRC),
|
| 92 |
+
created=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
| 93 |
+
changes=['h2_pour_delta.py: unchanged original 78-D H2 model (no joint velocity)',
|
| 94 |
+
'hb_aug_v2.py: unchanged copy of the shared spec-v2 implementation',
|
| 95 |
+
'h2_hbaug.py: H2HBAugV2Dataset (aug v2 + 31-joint proprio aug aligned with GR00T A20: per-sample '
|
| 96 |
+
'perturb_joints, no delay, no history drop; 78-D state, no velocity)',
|
| 97 |
+
'h2_pour.py: total frame count read from meta/info.json (was the A18 constant 72752) so A17 loads',
|
| 98 |
+
'train_deltaflow.py: shared 40-epoch entry, full state at every save, --resume'],
|
| 99 |
+
sha256={rel: sha(tmp / rel) for rel in files}))
|
| 100 |
+
tmp.rename(CODE)
|
| 101 |
+
log('H2 code copy built:', CODE)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def prepare(task):
|
| 105 |
+
spec = TASKS[task]
|
| 106 |
+
DATA, FRAMES = spec['data'], spec['frames']
|
| 107 |
+
R = ROOT / task
|
| 108 |
+
if (R / 'prepared.ok').is_file():
|
| 109 |
+
log(task, 'already prepared, skipping')
|
| 110 |
+
return
|
| 111 |
+
if R.exists():
|
| 112 |
+
aside = ROOT / f'{task}.incomplete-{time.strftime("%Y%m%dT%H%M%S")}'
|
| 113 |
+
R.rename(aside)
|
| 114 |
+
log(task, 'half-prepared directory moved to', aside)
|
| 115 |
+
(R / 'run').mkdir(parents=True)
|
| 116 |
+
(R / 'stats').mkdir()
|
| 117 |
+
sys.path.insert(0, str(CODE / 'src'))
|
| 118 |
+
os.environ.setdefault('HF_HUB_OFFLINE', '1')
|
| 119 |
+
from imagewam.datasets import h2_pour
|
| 120 |
+
if not h2_pour.__file__.startswith(str(CODE / 'src')):
|
| 121 |
+
raise SystemExit(f'imported the wrong imagewam: {h2_pour.__file__}')
|
| 122 |
+
if h2_pour.STATE_DIM != STATE or h2_pour.ACTION_DIM != ACTION:
|
| 123 |
+
raise SystemExit(f'h2_pour dims {h2_pour.STATE_DIM}/{h2_pour.ACTION_DIM}, expected {STATE}/{ACTION}')
|
| 124 |
+
episodes, instruction, names = h2_pour.load_collection(DATA)
|
| 125 |
+
lengths = [int(e['length']) for e in episodes]
|
| 126 |
+
if [e['episode_index'] for e in episodes] != list(range(EPISODES)) or sum(lengths) != FRAMES:
|
| 127 |
+
raise SystemExit(f'{task} collection is not episodes 0-99 / {FRAMES} frames')
|
| 128 |
+
if list(h2_pour.CAMERAS) != ['observation.images.head', 'observation.images.chest']:
|
| 129 |
+
raise SystemExit(f'Unexpected H2 cameras {h2_pour.CAMERAS}')
|
| 130 |
+
|
| 131 |
+
log(task, 'normalization (q01/q99 of state78 / action75 over episodes 0-99)')
|
| 132 |
+
h2_pour.fit(DATA, R / 'stats/normalization.json')
|
| 133 |
+
|
| 134 |
+
u = math.ceil(FRAMES / (base.WORLD * base.MICRO))
|
| 135 |
+
total, save_steps, warmup, n_epochs = base.fixed_schedule(u)
|
| 136 |
+
prompt = h2_pour.DEFAULT_PROMPT.format(task=instruction)
|
| 137 |
+
qinfo = None
|
| 138 |
+
if spec['qwen'] == 'copy':
|
| 139 |
+
shutil.copytree(OLD / 'qwen-text', R / 'qwen-text')
|
| 140 |
+
else:
|
| 141 |
+
import qwen_cache
|
| 142 |
+
ref_episodes, ref_instruction, _ = h2_pour.load_collection(TASKS['a18']['data'])
|
| 143 |
+
_, qinfo = qwen_cache.make(prompt, R / 'qwen-text', CODE, base.QWEN3, base.FLUX2_SRC,
|
| 144 |
+
h2_pour.DEFAULT_PROMPT.format(task=ref_instruction), OLD / 'qwen-text')
|
| 145 |
+
log(task, 'qwen cache computed', qinfo)
|
| 146 |
+
cache = R / 'qwen-text' / f'{hashlib.sha256(prompt.encode()).hexdigest()}.qwen3_flux2_len128.pt'
|
| 147 |
+
if not cache.is_file():
|
| 148 |
+
raise SystemExit(f'{task}: instruction cache missing: {cache.name}')
|
| 149 |
+
|
| 150 |
+
source_yaml = OLD / 'run/production.yaml'
|
| 151 |
+
cfg = yaml.safe_load(source_yaml.read_text())
|
| 152 |
+
cfg.update(output_dir=str(R / 'run/production'), num_epochs=n_epochs, max_steps=total,
|
| 153 |
+
lr_scheduler_total_steps=total, warmup_steps=warmup, save_every=0,
|
| 154 |
+
save_steps=save_steps, save_at_end=True, keep_latest_weights=base.KEEP_CKPTS, keep_latest_state_only=True,
|
| 155 |
+
resume=None, eval_every=0)
|
| 156 |
+
if cfg['batch_size'] != base.MICRO or cfg['gradient_accumulation_steps'] != 1 or cfg['data']['val'] is not None:
|
| 157 |
+
raise SystemExit(f'{task}: unexpected batch / accumulation / validation in the H2 config')
|
| 158 |
+
cfg['wandb'].update(enabled=False, mode='disabled', name=f'deltaflow-{task}-40ep-augv2-aligned',
|
| 159 |
+
group='h2-action75-state78')
|
| 160 |
+
old_train = cfg['data']['train']
|
| 161 |
+
cfg['data']['train'] = dict(
|
| 162 |
+
_target_=DATASET_TARGET, dataset_dir=str(DATA), stats_path=str(R / 'stats/normalization.json'),
|
| 163 |
+
qwen_text_cache_dir=str(R / 'qwen-text'), qwen_context_len=old_train['qwen_context_len'],
|
| 164 |
+
qwen_text_dim=old_train['qwen_text_dim'], video_cache_frames=64,
|
| 165 |
+
image_augmentation=base.augmentation_config(),
|
| 166 |
+
proprio_augmentation=dict(
|
| 167 |
+
enabled=True, seed=20261106, joint_bias_degrees=2.0, joint_noise_std_degrees=0.25, control_hz=30,
|
| 168 |
+
recording_hz=30, affected_angle_channels=list(range(BODY)), velocity_channels=[],
|
| 169 |
+
action_label_noise=False, body_joints='observation.state[0:31] incl. head',
|
| 170 |
+
start_pose_library='not_applied_offline', render_randomization='not_applied_offline', **ALIGNED))
|
| 171 |
+
m = cfg['model']
|
| 172 |
+
if (m['expected_action_dim'], m['action_dit_config']['action_dim'], len(m['loss']['action_a2a']['source_noise_dim_mask']),
|
| 173 |
+
m['proprio_dim'], m['action_state_dim'], m['expected_action_horizon']) != (ACTION, ACTION, ACTION, STATE, STATE, 16):
|
| 174 |
+
raise SystemExit(f'{task}: the H2 config is not the 78-D state / 75-D action Delta-flow it should be')
|
| 175 |
+
if 'h2_pour_delta' not in str(m.get('_target_', '')):
|
| 176 |
+
raise SystemExit(f"{task}: unexpected model target {m.get('_target_')!r} (want the original h2_pour_delta)")
|
| 177 |
+
cfg['wholebody_contract'] = dict(
|
| 178 |
+
task=task, robot='h2_sharpa', world_size=base.WORLD, zero_stage=1, fresh_base=True,
|
| 179 |
+
production_output_dir=str(R / 'run/production'), updates_per_epoch=u, train_episode_ids='0-99',
|
| 180 |
+
statistics_sha256=sha(R / 'stats/normalization.json'), augmentation=base.AUG_VERSION,
|
| 181 |
+
proprio_augmentation='aligned with GR00T A20 (per-sample perturb_joints, no delay, no history drop)',
|
| 182 |
+
state_layout='joints75 + gravity3 = 78 (no velocity)',
|
| 183 |
+
action_layout='body_target31 + hands44 = 75', cameras=list(h2_pour.CAMERAS),
|
| 184 |
+
measured_loader=dict(dataset_frames=FRAMES, raw_loader_batches=math.ceil(FRAMES / base.MICRO),
|
| 185 |
+
per_rank_microbatches=u, world_size=base.WORLD, microbatch=base.MICRO,
|
| 186 |
+
gradient_accumulation=1),
|
| 187 |
+
source_config=str(source_yaml))
|
| 188 |
+
for value in (cfg['output_dir'], cfg['data']['train']['stats_path'], cfg['data']['train']['qwen_text_cache_dir']):
|
| 189 |
+
if value.startswith(str(OLD) + '/'):
|
| 190 |
+
raise SystemExit(f'{task}: config still points into the old experiment: {value}')
|
| 191 |
+
base.no_velocity(cfg, task)
|
| 192 |
+
(R / 'run/production.yaml').write_text(yaml.safe_dump(cfg, sort_keys=False), encoding='utf-8')
|
| 193 |
+
previous = OLD / 'normalization.json'
|
| 194 |
+
dump(R / 'data-verified.json', dict(
|
| 195 |
+
task=task, dataset_dir=str(DATA), dataset_source=spec['source'], total_episodes_in_collection=EPISODES,
|
| 196 |
+
episodes=EPISODES, train_episode_ids=list(range(EPISODES)), shelved_episode_ids=[], frames=FRAMES,
|
| 197 |
+
updates_per_epoch=u, epochs=None, fixed_steps=True,
|
| 198 |
+
epochs_equivalent=round(total / u, 2), max_steps=total, warmup_steps=warmup,
|
| 199 |
+
save_steps=save_steps, control_hz=30, recording_hz=30, statistics_sha256=sha(R / 'stats/normalization.json'),
|
| 200 |
+
statistics_equal_previous_a18_run=(task == 'a18' and previous.is_file()
|
| 201 |
+
and sha(previous) == sha(R / 'stats/normalization.json')),
|
| 202 |
+
augmentation=base.AUG_VERSION, proprio_augmentation=ALIGNED, qwen_cache=(qinfo or 'copied from the A18 run'),
|
| 203 |
+
state_dim=STATE, action_dim=ACTION, joint_velocity=False,
|
| 204 |
+
code=str(CODE), code_record_sha256=sha(CODE / 'deltaflow-code.json')))
|
| 205 |
+
|
| 206 |
+
log(task, 'model module check (no weights loaded)')
|
| 207 |
+
import imagewam.h2_pour_delta as model_module
|
| 208 |
+
factory = inspect.signature(model_module.create_imagewam_flux2_wholebody_delta)
|
| 209 |
+
if (factory.parameters['action_state_dim'].default != STATE
|
| 210 |
+
or factory.parameters['expected_action_dim'].default != ACTION):
|
| 211 |
+
raise SystemExit(f'{task}: original H2 model module is not 78-D state / 75-D action')
|
| 212 |
+
|
| 213 |
+
log(task, 'loading real augmented samples (CPU)')
|
| 214 |
+
import numpy as np
|
| 215 |
+
import torch
|
| 216 |
+
from hydra.utils import instantiate
|
| 217 |
+
from omegaconf import OmegaConf
|
| 218 |
+
dataset = instantiate(OmegaConf.load(R / 'run/production.yaml').data.train)
|
| 219 |
+
if len(dataset) != FRAMES or list(dataset.episode_ids) != list(range(EPISODES)):
|
| 220 |
+
raise SystemExit(f'{task}: dataset length {len(dataset)} differs from {FRAMES}')
|
| 221 |
+
first, sample = dataset[0], dataset[145]
|
| 222 |
+
if (tuple(first['video'].shape) != (3, 2, 224, 448) or tuple(first['proprio'].shape) != (16, STATE)
|
| 223 |
+
or tuple(sample['proprio'].shape) != (16, STATE) or tuple(sample['proprio_dim_is_pad'].shape) != (STATE,)
|
| 224 |
+
or tuple(sample['action'].shape) != (16, ACTION) or tuple(sample['past_action'].shape) != (16, ACTION)
|
| 225 |
+
or not torch.isfinite(sample['proprio']).all() or dataset.last_augmentation['visual']['boost'] is None):
|
| 226 |
+
raise SystemExit(f'{task}: sample schema check failed')
|
| 227 |
+
perturb = []
|
| 228 |
+
for i in range(0, FRAMES, max(1, FRAMES // 40)):
|
| 229 |
+
dataset[i]
|
| 230 |
+
num = dataset.last_augmentation['numeric']
|
| 231 |
+
if num['delay_recorded_frames'] != 0 or num['history_drop_rows'] or num['state_frame'] != num['anchor_frame']:
|
| 232 |
+
raise SystemExit(f'{task}: delay / history drop still active: {num}')
|
| 233 |
+
perturb.append(num['joint_perturbation_degrees'])
|
| 234 |
+
perturb = np.asarray(perturb)
|
| 235 |
+
if perturb.shape[1] != BODY or np.abs(perturb).max() > 2.0 + 0.75 + 1e-3 or perturb.std(axis=0).min() < 0.5:
|
| 236 |
+
raise SystemExit(f'{task}: joint perturbation is not per-sample +-2 deg + 0.25 deg noise')
|
| 237 |
+
dump(R / 'prepared.ok', dict(frames=FRAMES, updates_per_epoch=u, max_steps=total, save_steps=save_steps,
|
| 238 |
+
proprio_check=dict(samples=len(perturb), max_abs_deg=float(np.abs(perturb).max()),
|
| 239 |
+
std_deg=float(perturb.std()))))
|
| 240 |
+
log(task, 'PREPARED', json.dumps(dict(frames=FRAMES, updates_per_epoch=u, max_steps=total, save_steps=save_steps,
|
| 241 |
+
warmup=warmup, state_dim=STATE, action_dim=ACTION)))
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
if __name__ == '__main__':
|
| 245 |
+
tasks = sys.argv[1:] or list(TASKS)
|
| 246 |
+
for t in tasks:
|
| 247 |
+
if t not in TASKS:
|
| 248 |
+
raise SystemExit(f'unknown H2 task {t}; choose from {list(TASKS)}')
|
| 249 |
+
build_code()
|
| 250 |
+
for t in tasks:
|
| 251 |
+
prepare(t)
|
code/hb_deltaflow/qwen_cache.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FLUX.2 Qwen3 instruction cache for Delta-flow tasks that have no previous experiment to copy it from.
|
| 2 |
+
|
| 3 |
+
Same computation as HumanWAM scripts/flux2/precompute_flux2_qwen3_embeds.py (chat template, enable_thinking False,
|
| 4 |
+
padding to max_length 128, hidden states of flux2.text_encoder.OUTPUT_LAYERS_QWEN3 concatenated, stored as bf16 +
|
| 5 |
+
bool mask), on CPU in bfloat16 (the GPU original also runs the model in bfloat16). make() first recomputes a reference
|
| 6 |
+
prompt whose cache already exists (A3) and refuses to write anything unless the result matches it.
|
| 7 |
+
"""
|
| 8 |
+
import hashlib
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
_MODEL = {}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _load(code, qwen_path, flux2_src):
|
| 19 |
+
if "m" not in _MODEL:
|
| 20 |
+
sys.path.insert(0, str(Path(code) / "src"))
|
| 21 |
+
from imagewam.models.backbones.flux2_imports import ensure_flux2_importable
|
| 22 |
+
ensure_flux2_importable(str(flux2_src))
|
| 23 |
+
from flux2.text_encoder import OUTPUT_LAYERS_QWEN3
|
| 24 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 25 |
+
torch.set_num_threads(min(32, os.cpu_count() or 8))
|
| 26 |
+
tok = AutoTokenizer.from_pretrained(str(qwen_path))
|
| 27 |
+
model = AutoModelForCausalLM.from_pretrained(str(qwen_path), torch_dtype=torch.bfloat16).eval()
|
| 28 |
+
_MODEL["m"] = (tok, model, tuple(OUTPUT_LAYERS_QWEN3))
|
| 29 |
+
return _MODEL["m"]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def embed(prompt, code, qwen_path, flux2_src, context_len=128):
|
| 33 |
+
tok, model, layers = _load(code, qwen_path, flux2_src)
|
| 34 |
+
rendered = tok.apply_chat_template([{"role": "user", "content": prompt}], tokenize=False,
|
| 35 |
+
add_generation_prompt=True, enable_thinking=False)
|
| 36 |
+
enc = tok([rendered], return_tensors="pt", padding="max_length", truncation=True, max_length=context_len)
|
| 37 |
+
with torch.no_grad():
|
| 38 |
+
out = model(input_ids=enc.input_ids, attention_mask=enc.attention_mask, output_hidden_states=True,
|
| 39 |
+
use_cache=False)
|
| 40 |
+
hidden = torch.stack([out.hidden_states[k] for k in layers], dim=1) # [1, c, l, d]
|
| 41 |
+
hidden = hidden.permute(0, 2, 1, 3).reshape(1, context_len, -1) # b l (c d)
|
| 42 |
+
return dict(text_hidden_states=hidden[0].to(torch.bfloat16).clone(),
|
| 43 |
+
text_attention_mask=enc.attention_mask[0].to(torch.bool).clone())
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def cache_name(prompt, context_len=128):
|
| 47 |
+
return f"{hashlib.sha256(prompt.encode('utf-8')).hexdigest()}.qwen3_flux2_len{context_len}.pt"
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def check_reference(ref_prompt, ref_dir, code, qwen_path, flux2_src):
|
| 51 |
+
ref = torch.load(Path(ref_dir) / cache_name(ref_prompt), map_location="cpu", weights_only=True)
|
| 52 |
+
new = embed(ref_prompt, code, qwen_path, flux2_src)
|
| 53 |
+
if not torch.equal(ref["text_attention_mask"].bool(), new["text_attention_mask"]):
|
| 54 |
+
raise SystemExit("qwen cache: attention mask differs from the reference cache")
|
| 55 |
+
m = ref["text_attention_mask"].bool()
|
| 56 |
+
a, b = ref["text_hidden_states"][m].float(), new["text_hidden_states"][m].float()
|
| 57 |
+
cos = torch.nn.functional.cosine_similarity(a, b, dim=-1)
|
| 58 |
+
rel = ((a - b).norm() / a.norm()).item()
|
| 59 |
+
info = dict(min_token_cosine=float(cos.min()), mean_token_cosine=float(cos.mean()), relative_l2=rel,
|
| 60 |
+
tokens=int(m.sum()))
|
| 61 |
+
if info["min_token_cosine"] < 0.99 or rel > 0.05:
|
| 62 |
+
raise SystemExit(f"qwen cache: CPU recomputation does not match the reference cache: {info}")
|
| 63 |
+
return info
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def make(prompt, out_dir, code, qwen_path, flux2_src, ref_prompt, ref_dir):
|
| 67 |
+
out_dir = Path(out_dir)
|
| 68 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 69 |
+
path = out_dir / cache_name(prompt)
|
| 70 |
+
if path.is_file():
|
| 71 |
+
return path, None
|
| 72 |
+
info = check_reference(ref_prompt, ref_dir, code, qwen_path, flux2_src)
|
| 73 |
+
payload = embed(prompt, code, qwen_path, flux2_src)
|
| 74 |
+
if not torch.isfinite(payload["text_hidden_states"].float()).all() or not payload["text_attention_mask"].any():
|
| 75 |
+
raise SystemExit("qwen cache: invalid embedding")
|
| 76 |
+
tmp = out_dir / f".{path.name}.tmp{os.getpid()}"
|
| 77 |
+
torch.save(payload, tmp)
|
| 78 |
+
os.replace(tmp, path)
|
| 79 |
+
return path, info
|
code/hb_deltaflow/train_deltaflow.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ImageWAM Delta-flow 40-epoch training entry (G1 B6/A3/A7 and H2 A18, 100 episodes, augmentation v2).
|
| 2 |
+
|
| 3 |
+
Installed by prepare_deltaflow.py / prepare_deltaflow_a18.py as <code>/scripts/wholebody/train_deltaflow.py.
|
| 4 |
+
Derived from train_a3a7.py:
|
| 5 |
+
- same native runtime/trainer, same 32-GPU loader-length checks and finish receipts;
|
| 6 |
+
- no separate 1-update smoke phase (the supervisor retries / resumes instead);
|
| 7 |
+
- every save (epoch 10/20/30/40) writes weights + full training state; the trainer keeps all four weight files
|
| 8 |
+
(keep_latest_weights=4) and only the newest training state (keep_latest_state_only=true);
|
| 9 |
+
- --resume <run/production/checkpoints/state/step_N> continues an interrupted run from its newest state.
|
| 10 |
+
"""
|
| 11 |
+
import argparse
|
| 12 |
+
import json
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
from omegaconf import OmegaConf
|
| 16 |
+
from imagewam import runtime
|
| 17 |
+
from imagewam.trainer import Wan22Trainer
|
| 18 |
+
|
| 19 |
+
# dataset class -> required train_split_mode (None: the loader itself admits exactly episodes 0-99)
|
| 20 |
+
DATASET_TARGETS = {
|
| 21 |
+
'imagewam.datasets.wholebody_hbaug.WholeBodyHBAugV2LeRobotDataset': 'first_100', # G1: B6 / A3 / A7
|
| 22 |
+
'imagewam.datasets.h2_hbaug.H2HBAugV2Dataset': None, # H2: A18
|
| 23 |
+
}
|
| 24 |
+
AUG_VERSION = 'hb-aug-v2-20261109'
|
| 25 |
+
EPOCHS, SAVE_EVERY_EPOCHS, WARMUP_EPOCHS, WORLD = 40, 10, 2, 32
|
| 26 |
+
EXPECTED = {}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def require(condition, message):
|
| 30 |
+
if not condition:
|
| 31 |
+
raise RuntimeError(message)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class DeltaFlowTrainer(Wan22Trainer):
|
| 35 |
+
def _resume_or_load_checkpoint(self):
|
| 36 |
+
require(len(self.train_dataset) == EXPECTED['frames'] and len(self.train_loader) == EXPECTED['updates_per_epoch']
|
| 37 |
+
and self.accelerator.num_processes == WORLD,
|
| 38 |
+
'Expected the pinned 100-episode collection, its updates/epoch and 32 GPUs')
|
| 39 |
+
require(type(self.train_dataset).__name__ in {t.rsplit('.', 1)[1] for t in DATASET_TARGETS},
|
| 40 |
+
'Augmentation v2 dataset missing')
|
| 41 |
+
return super()._resume_or_load_checkpoint()
|
| 42 |
+
|
| 43 |
+
def train(self):
|
| 44 |
+
start = int(self.global_step)
|
| 45 |
+
super().train()
|
| 46 |
+
engine = self.accelerator.deepspeed_engine_wrapped.engine
|
| 47 |
+
require(self.global_step == self.max_steps and engine.skipped_steps == 0,
|
| 48 |
+
'Training ended without reaching its optimizer-update target')
|
| 49 |
+
if not self.resume:
|
| 50 |
+
require(engine.global_steps == self.global_step, 'Trainer and optimizer update counts differ')
|
| 51 |
+
rank = self.accelerator.process_index
|
| 52 |
+
path = Path(self.output_dir) / f'finish-rank-{rank:02d}.json'
|
| 53 |
+
path.write_text(json.dumps({'rank': rank, 'global_step': self.global_step,
|
| 54 |
+
'optimizer_steps': int(engine.global_steps), 'resumed_from_step': start}) + '\n')
|
| 55 |
+
self.accelerator.wait_for_everyone()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def check_config(cfg, root, resume):
|
| 59 |
+
receipt = json.loads((root / 'data-verified.json').read_text())
|
| 60 |
+
u = int(receipt['updates_per_epoch'])
|
| 61 |
+
total = EPOCHS * u
|
| 62 |
+
EXPECTED.update(frames=int(receipt['frames']), updates_per_epoch=u)
|
| 63 |
+
d = cfg.data.train
|
| 64 |
+
require(d._target_ in DATASET_TARGETS and d.image_augmentation.version == AUG_VERSION
|
| 65 |
+
and d.image_augmentation.enabled is True, 'Config does not use augmentation v2')
|
| 66 |
+
split_mode = DATASET_TARGETS[d._target_]
|
| 67 |
+
require((split_mode is None or d.train_split_mode == split_mode) and receipt['train_episode_ids'] == list(range(100)),
|
| 68 |
+
'Training must use exactly episodes 0-99')
|
| 69 |
+
require(cfg.num_epochs == EPOCHS and cfg.max_steps == total and cfg.lr_scheduler_total_steps == total
|
| 70 |
+
and cfg.warmup_steps == WARMUP_EPOCHS * u, 'Epoch/step schedule differs from 40 x updates_per_epoch')
|
| 71 |
+
require(list(cfg.save_steps) == [e * u for e in range(SAVE_EVERY_EPOCHS, EPOCHS + 1, SAVE_EVERY_EPOCHS)]
|
| 72 |
+
and cfg.save_every == 0 and cfg.save_at_end is True and cfg.keep_latest_weights == 4
|
| 73 |
+
and cfg.keep_latest_state_only is True, 'Checkpoint policy differs (epochs 10/20/30/40, newest state only)')
|
| 74 |
+
output = Path(cfg.output_dir).resolve()
|
| 75 |
+
require(output == (root / 'run' / 'production').resolve(), 'Unexpected output directory')
|
| 76 |
+
require(cfg.resume is None, 'Pass --resume on the command line, not in the YAML')
|
| 77 |
+
if resume is None:
|
| 78 |
+
weights = list((output / 'checkpoints' / 'weights').glob('*.pt'))
|
| 79 |
+
require(not weights, 'Fresh start into an output directory that already holds weights')
|
| 80 |
+
return
|
| 81 |
+
state = Path(resume).resolve()
|
| 82 |
+
require(state.parent == output / 'checkpoints' / 'state' and (state / 'trainer_state.json').is_file(),
|
| 83 |
+
f'Invalid resume state {state}')
|
| 84 |
+
cfg.resume = str(state)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
if __name__ == '__main__':
|
| 88 |
+
parser = argparse.ArgumentParser()
|
| 89 |
+
parser.add_argument('--config', type=Path, required=True)
|
| 90 |
+
parser.add_argument('--resume', default=None)
|
| 91 |
+
args = parser.parse_args()
|
| 92 |
+
config = args.config.resolve()
|
| 93 |
+
cfg = OmegaConf.load(config)
|
| 94 |
+
check_config(cfg, config.parent.parent, args.resume)
|
| 95 |
+
runtime.Wan22Trainer = DeltaFlowTrainer
|
| 96 |
+
runtime.run_training(cfg)
|
code/hb_deltaflow/verify_cpu.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU verification of the Delta-flow 40-epoch package (run on a node after prepare_deltaflow.py; no GPU used).
|
| 2 |
+
|
| 3 |
+
$P/venv/bin/python $G/hb_deltaflow/verify_cpu.py # aug checks + b6 a3 a7
|
| 4 |
+
$P/venv/bin/python $G/hb_deltaflow/verify_cpu.py b6 --samples 48
|
| 5 |
+
$P/venv/bin/python $G/hb_deltaflow/verify_cpu.py --aug-only # before prepare: hb_aug_v2 from $G/hb_groot
|
| 6 |
+
Checks
|
| 7 |
+
1. hb_aug_v2 parameter statistics: each of the 15 effects is the 1.5x boost ~1/15 of the time, trigger rates match
|
| 8 |
+
the spec, 1x and 1.5x ranges are respected and the boosted range is actually reached.
|
| 9 |
+
2. hb_aug_v2.apply per effect in isolation: every effect changes the observation frame; colour / rotation-crop /
|
| 10 |
+
light / white balance also change the t+16 target, sensor/optics effects leave the target untouched.
|
| 11 |
+
3. per task: manifest / statistics cover exactly episodes 0-99 (B6 shelves 100-148), YAML has the
|
| 12 |
+
40-epoch schedule and epoch 10/20/30/40 saves and NO joint velocity anywhere (user rule 2026-11-10), then one
|
| 13 |
+
real augmented batch: shapes, labels unchanged, target frame = colour/geometry/light/WB only, proprio 44-D
|
| 14 |
+
(G1: body 29 + hands 12 + gravity 3) / 78-D (H2: 75 joints + gravity 3) with untouched hands/gravity when not
|
| 15 |
+
delayed.
|
| 16 |
+
"""
|
| 17 |
+
import argparse
|
| 18 |
+
import json
|
| 19 |
+
import math
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
import sys
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
import torch
|
| 25 |
+
|
| 26 |
+
H = Path(__file__).resolve().parent
|
| 27 |
+
G = H.parent
|
| 28 |
+
CODE = G / 'deltaflow-runs' / 'HumanWAM'
|
| 29 |
+
CODE_H2 = G / 'deltaflow-runs' / 'HumanWAM-h2'
|
| 30 |
+
ROOT = G / 'deltaflow-runs'
|
| 31 |
+
CONTROL_HZ = {'b6': 50, 'a3': 30, 'a7': 30}
|
| 32 |
+
TOTAL_EPISODES = {'b6': 149, 'a3': 100, 'a7': 100}
|
| 33 |
+
# src/imagewam/wholebody_delta.py of the 44-D reference run $P/real30-from-sim60-20261108/HumanWAM
|
| 34 |
+
REF_DELTA_SHA256 = '523244263496679f7046035f5df3d34590b7086e25173938d8a175dad963b9f3'
|
| 35 |
+
FAILURES = []
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def check(condition, message):
|
| 39 |
+
print((' ok ' if condition else ' FAIL ') + message, flush=True)
|
| 40 |
+
if not condition:
|
| 41 |
+
FAILURES.append(message)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def velocity_entries(value, path='cfg'):
|
| 45 |
+
"""Config entries that would add joint velocity (non-empty velocity_* settings or velocity file references)."""
|
| 46 |
+
found = []
|
| 47 |
+
if isinstance(value, dict):
|
| 48 |
+
for key, item in value.items():
|
| 49 |
+
if 'velocity' in str(key).lower() and item not in ([], None, False, 0):
|
| 50 |
+
found.append(f'{path}.{key}')
|
| 51 |
+
found += velocity_entries(item, f'{path}.{key}')
|
| 52 |
+
elif isinstance(value, list):
|
| 53 |
+
for i, item in enumerate(value):
|
| 54 |
+
found += velocity_entries(item, f'{path}[{i}]')
|
| 55 |
+
elif isinstance(value, str) and 'velocity' in value.lower() and 'no velocity' not in value.lower():
|
| 56 |
+
found.append(path)
|
| 57 |
+
return found
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
SENSOR = ('noise', 'gaussian_blur', 'jpeg', 'motion_blur', 'sharpen', 'defocus')
|
| 61 |
+
PROB = {'noise': .5, 'gaussian_blur': .15, 'jpeg': .35, 'motion_blur': .15, 'sharpen': .15, 'defocus': .10,
|
| 62 |
+
'light_shadow': .30, 'wb_channel': .30}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def active(p, name):
|
| 66 |
+
value = p[name]
|
| 67 |
+
return value is not None and value != 0.0
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def check_parameters(A, n=15000):
|
| 71 |
+
print('[1] hb_aug_v2 parameter statistics, n=%d' % n)
|
| 72 |
+
check(len(A.EFFECTS) == 15 and A.VERSION == 'hb-aug-v2-20261109', f'15 effects, version {A.VERSION}')
|
| 73 |
+
rng = np.random.default_rng(1234)
|
| 74 |
+
draws = [A.sample(rng, 2, 224) for _ in range(n)]
|
| 75 |
+
counts = {e: sum(p['boost'] == e for p in draws) for e in A.EFFECTS}
|
| 76 |
+
band = 5 * math.sqrt(n / 15)
|
| 77 |
+
check(all(abs(c - n / 15) < band for c in counts.values()), f'boost uniform over 15 effects: {counts}')
|
| 78 |
+
for name, prob in PROB.items():
|
| 79 |
+
rate = sum(active(p, name) for p in draws) / n
|
| 80 |
+
check(abs(rate - prob) < 0.03, f'{name} trigger rate {rate:.3f} (spec {prob})')
|
| 81 |
+
|
| 82 |
+
def span(name, value_fn, lo1, hi1, lo15, hi15):
|
| 83 |
+
normal = [v for p in draws if p['boost'] != name for v in value_fn(p)]
|
| 84 |
+
boosted = [v for p in draws if p['boost'] == name for v in value_fn(p)]
|
| 85 |
+
ok = (normal and boosted and lo1 - 1e-9 <= min(normal) and max(normal) <= hi1 + 1e-9
|
| 86 |
+
and lo15 - 1e-9 <= min(boosted) and max(boosted) <= hi15 + 1e-9
|
| 87 |
+
and (min(boosted) < lo1 - 1e-6 or max(boosted) > hi1 + 1e-6))
|
| 88 |
+
check(bool(ok), f'{name}: x1 in [{lo1},{hi1}], x1.5 in [{lo15},{hi15}] and beyond x1 '
|
| 89 |
+
f'(seen x1 {min(normal):.4g}..{max(normal):.4g}, x1.5 {min(boosted):.4g}..{max(boosted):.4g})')
|
| 90 |
+
|
| 91 |
+
span('exposure', lambda p: [p['exposure']], -.22, .22, -.33, .33)
|
| 92 |
+
span('contrast', lambda p: [p['contrast']], .88, 1.12, .82, 1.18)
|
| 93 |
+
span('saturation', lambda p: [p['saturation']], .76, 1.24, .64, 1.36)
|
| 94 |
+
span('hue', lambda p: [p['hue']], -.015, .015, -.0225, .0225)
|
| 95 |
+
span('gamma', lambda p: [p['gamma']], .89, 1.11, .835, 1.165)
|
| 96 |
+
span('temperature', lambda p: [p['temperature']], -.066, .066, -.099, .099)
|
| 97 |
+
span('rotation_crop', lambda p: [a for a, _ in p['geometry']], -1.5, 1.5, -2.25, 2.25)
|
| 98 |
+
span('rotation_crop', lambda p: [s - 1 - abs(a) * .015 for a, s in p['geometry']], 0, .02, 0, .03)
|
| 99 |
+
span('noise', lambda p: [p['noise']] if p['noise'] else [], 4 / 255, 4 / 255, 6 / 255, 6 / 255)
|
| 100 |
+
span('gaussian_blur', lambda p: [p['gaussian_blur']] if p['gaussian_blur'] else [], .25, .65, .375, .975)
|
| 101 |
+
span('jpeg', lambda p: [p['jpeg']] if p['jpeg'] is not None else [], 70, 95, 55, 92)
|
| 102 |
+
span('motion_blur', lambda p: [p['motion_blur'][0]] if p['motion_blur'] else [], 3, 7, 4.5, 10.5)
|
| 103 |
+
span('sharpen', lambda p: [p['sharpen']] if p['sharpen'] else [], .2, .6, .3, .9)
|
| 104 |
+
span('defocus', lambda p: [p['defocus']] if p['defocus'] else [], 1, 2, 1.5, 3)
|
| 105 |
+
span('light_shadow', lambda p: [b[3] for cam in (p['light_shadow'] or []) for b in cam], .7, 1.3, .55, 1.45)
|
| 106 |
+
span('wb_channel', lambda p: [g for cam in (p['wb_channel'] or []) for g in cam if g != 1.0], .9, 1.1, .85, 1.15)
|
| 107 |
+
angles = [p['motion_blur'][1] for p in draws if p['motion_blur']]
|
| 108 |
+
check(min(angles) >= 0 and max(angles) <= 180, 'motion blur direction in 0-180 deg')
|
| 109 |
+
blobs = [cam for p in draws if p['light_shadow'] for cam in p['light_shadow']]
|
| 110 |
+
check(all(1 <= len(c) <= 2 and all(.15 <= b[2] <= .35 for b in c) for c in blobs),
|
| 111 |
+
'light: 1-2 patches per camera, size 15-35 % of the frame')
|
| 112 |
+
wb = [cam for p in draws if p['wb_channel'] for cam in p['wb_channel']]
|
| 113 |
+
check(all(sum(g != 1.0 for g in cam) <= 1 for cam in wb), 'white balance: one channel per camera')
|
| 114 |
+
geometry_differs = sum(p['geometry'][0] != p['geometry'][1] for p in draws) / n
|
| 115 |
+
check(geometry_differs > .99, 'rotation/crop drawn independently per camera')
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def identity(views=2):
|
| 119 |
+
return {'boost': None, 'exposure': 0.0, 'contrast': 1.0, 'saturation': 1.0, 'hue': 0.0, 'gamma': 1.0,
|
| 120 |
+
'temperature': 0.0, 'geometry': [(0.0, 1.0)] * views, 'noise': 0.0, 'gaussian_blur': 0.0, 'jpeg': None,
|
| 121 |
+
'motion_blur': None, 'sharpen': 0.0, 'defocus': 0.0, 'light_shadow': None, 'wb_channel': None}
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def test_image():
|
| 125 |
+
yy, xx = torch.meshgrid(torch.linspace(0, 1, 224), torch.linspace(0, 1, 224), indexing='ij')
|
| 126 |
+
gen = torch.Generator().manual_seed(7)
|
| 127 |
+
frames = []
|
| 128 |
+
for t in range(2):
|
| 129 |
+
cams = []
|
| 130 |
+
for v in range(2):
|
| 131 |
+
base = torch.stack([0.2 + 0.6 * xx, 0.2 + 0.6 * yy, 0.5 + 0.3 * torch.sin(12 * xx + 7 * yy + t + v)])
|
| 132 |
+
checker = ((torch.floor(xx * 16) + torch.floor(yy * 16)) % 2) * 0.15
|
| 133 |
+
cams.append((base * 0.8 + checker + 0.05 * torch.rand(3, 224, 224, generator=gen)).clamp(0, 1))
|
| 134 |
+
frames.append(torch.stack(cams))
|
| 135 |
+
return torch.stack(frames) # [T=2, V=2, 3, 224, 224]
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def check_apply(A):
|
| 139 |
+
print('[2] hb_aug_v2.apply: every effect fires, observation-only vs target rules')
|
| 140 |
+
x = test_image()
|
| 141 |
+
obs = torch.tensor([True, False])
|
| 142 |
+
ref = A.apply(x, identity(), obs)
|
| 143 |
+
check((ref - x).abs().max().item() < 2e-3, 'identity parameters leave the frames unchanged')
|
| 144 |
+
settings = {
|
| 145 |
+
'exposure': {'exposure': .3}, 'contrast': {'contrast': 1.15}, 'saturation': {'saturation': 1.3},
|
| 146 |
+
'hue': {'hue': .02}, 'gamma': {'gamma': 1.15}, 'temperature': {'temperature': .09},
|
| 147 |
+
'rotation_crop': {'geometry': [(2.0, 1.05), (-2.0, 1.04)]}, 'noise': {'noise': 6 / 255},
|
| 148 |
+
'gaussian_blur': {'gaussian_blur': .9}, 'jpeg': {'jpeg': 55}, 'motion_blur': {'motion_blur': (9.0, 30.0)},
|
| 149 |
+
'sharpen': {'sharpen': .8}, 'defocus': {'defocus': 2.5},
|
| 150 |
+
'light_shadow': {'light_shadow': [[(.5, .5, .25, 1.4)], [(.3, .6, .2, .6)]]},
|
| 151 |
+
'wb_channel': {'wb_channel': [[1.0, 1.12, 1.0], [0.88, 1.0, 1.0]]},
|
| 152 |
+
}
|
| 153 |
+
check(sorted(settings) == sorted(A.EFFECTS), 'all 15 effects exercised')
|
| 154 |
+
for name, change in settings.items():
|
| 155 |
+
p = dict(identity(), **change)
|
| 156 |
+
y = A.apply(x, p, obs, noise_seed=3)
|
| 157 |
+
d_obs = [(y[0, v] - ref[0, v]).abs().mean().item() for v in range(2)]
|
| 158 |
+
d_tgt = [(y[1, v] - ref[1, v]).abs().max().item() for v in range(2)]
|
| 159 |
+
fires = min(d_obs) > 5e-4
|
| 160 |
+
if name in SENSOR:
|
| 161 |
+
check(fires and max(d_tgt) < 1e-6, f'{name}: observation changed {min(d_obs):.4f}, target untouched')
|
| 162 |
+
else:
|
| 163 |
+
check(fires and min((y[1, v] - ref[1, v]).abs().mean().item() for v in range(2)) > 5e-4,
|
| 164 |
+
f'{name}: observation and t+16 target both changed')
|
| 165 |
+
p = dict(identity(), exposure=.3, saturation=1.3)
|
| 166 |
+
y = A.apply(x, p, obs)
|
| 167 |
+
same_colour = all(torch.allclose(y[t, v], A.apply(x[t:t + 1, v:v + 1], dict(p, geometry=[(0.0, 1.0)]),
|
| 168 |
+
torch.tensor([t == 0]))[0, 0], atol=1e-5)
|
| 169 |
+
for t in range(2) for v in range(2))
|
| 170 |
+
check(same_colour, 'one colour parameter set applied identically to both cameras and both times')
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def check_task(task, A, samples):
|
| 174 |
+
import yaml
|
| 175 |
+
from hydra.utils import instantiate
|
| 176 |
+
from omegaconf import OmegaConf
|
| 177 |
+
from torch.utils.data import default_collate
|
| 178 |
+
from imagewam.datasets.wholebody import WholeBodyLeRobotDataset
|
| 179 |
+
R = ROOT / task
|
| 180 |
+
print(f'[3] task {task}: {R}')
|
| 181 |
+
check((R / 'prepared.ok').is_file(), 'prepared.ok present')
|
| 182 |
+
receipt = json.loads((R / 'data-verified.json').read_text())
|
| 183 |
+
u, frames = receipt['updates_per_epoch'], receipt['frames']
|
| 184 |
+
train = list(range(100))
|
| 185 |
+
shelved = list(range(100, TOTAL_EPISODES[task]))
|
| 186 |
+
manifest = json.loads((R / 'split-manifest.json').read_text())
|
| 187 |
+
stats = json.loads((R / 'stats/dataset_stats.json').read_text())
|
| 188 |
+
check(manifest['train_episode_ids'] == train and manifest['validation_episode_ids'] == shelved,
|
| 189 |
+
f'split manifest trains 0-99, shelves {len(shelved)} episodes')
|
| 190 |
+
check(stats['train_episode_ids'] == train and stats['heldout_episode_ids'] == shelved
|
| 191 |
+
and stats['train_frames'] == frames and stats['train_split_mode'] == 'first_100',
|
| 192 |
+
f'normalization statistics fit on episodes 0-99 only ({frames} frames)')
|
| 193 |
+
check(not (R / 'stats/velocity.npz').exists() and not list(R.rglob('*velocity*')), 'no velocity cache / statistics files')
|
| 194 |
+
cfg = yaml.safe_load((R / 'run/production.yaml').read_text())
|
| 195 |
+
left = velocity_entries(cfg)
|
| 196 |
+
check(not left, f'config has no joint-velocity setting ({left or "none"})')
|
| 197 |
+
d = cfg['data']['train']
|
| 198 |
+
saves = [e * u for e in (10, 20, 30, 40)]
|
| 199 |
+
check(cfg['num_epochs'] == 40 and cfg['max_steps'] == cfg['lr_scheduler_total_steps'] == 40 * u
|
| 200 |
+
and cfg['warmup_steps'] == 2 * u and cfg['save_steps'] == saves and cfg['save_every'] == 0
|
| 201 |
+
and cfg['keep_latest_weights'] == 4 and cfg['keep_latest_state_only'] is True and cfg['save_at_end'] is True,
|
| 202 |
+
f'schedule: {u} updates/epoch, max_steps {40 * u}, warmup {2 * u}, saves {saves}, newest state only')
|
| 203 |
+
check(d['proprio_augmentation']['control_hz'] == CONTROL_HZ[task] and d['proprio_augmentation']['recording_hz'] == 30,
|
| 204 |
+
f'control {CONTROL_HZ[task]} Hz / record 30 Hz')
|
| 205 |
+
check(d['_target_'].endswith('WholeBodyHBAugV2LeRobotDataset')
|
| 206 |
+
and d['image_augmentation']['version'] == A.VERSION, 'dataset class uses hb_aug_v2')
|
| 207 |
+
check(cfg['model']['proprio_dim'] == cfg['model']['action_state_dim'] == 44 and cfg['model']['expected_action_dim'] == 76,
|
| 208 |
+
'model 44-D state (no velocity) / 76-D action, as reference real30-from-sim60-20261108')
|
| 209 |
+
check(d['_target_'].endswith('WholeBodyHBAugV2LeRobotDataset') and not d['proprio_augmentation'].get('velocity_channels'),
|
| 210 |
+
'loader = WholeBodyRobust-based HB aug v2, velocity_channels empty')
|
| 211 |
+
# the model factory itself (2026-11-10: config + data said 44 but the factory of the B6 60-epoch copy required 73)
|
| 212 |
+
import hashlib
|
| 213 |
+
import inspect
|
| 214 |
+
import re
|
| 215 |
+
import imagewam.wholebody_delta as model_module
|
| 216 |
+
src = Path(model_module.__file__).read_bytes()
|
| 217 |
+
sig = inspect.signature(model_module.create_imagewam_flux2_wholebody_delta)
|
| 218 |
+
check(cfg['model']['_target_'] == 'imagewam.wholebody_delta.create_imagewam_flux2_wholebody_delta'
|
| 219 |
+
and sig.parameters['action_state_dim'].default == 44 and 'state44' in model_module.SCHEMA_ID
|
| 220 |
+
and b'(None, 44)' in src and re.search(rb'\b73\b', src) is None
|
| 221 |
+
and hashlib.sha256(src).hexdigest() == REF_DELTA_SHA256,
|
| 222 |
+
'model factory wholebody_delta.py = state44 copy of real30-from-sim60-20261108 (no 73 anywhere)')
|
| 223 |
+
|
| 224 |
+
dataset = instantiate(OmegaConf.load(R / 'run/production.yaml').data.train)
|
| 225 |
+
check(len(dataset) == frames and list(dataset.episode_ids) == train, f'dataset = episodes 0-99, {len(dataset)} samples')
|
| 226 |
+
rng = np.random.default_rng(99)
|
| 227 |
+
index = sorted(set(rng.integers(0, len(dataset), samples).tolist()) | {0})
|
| 228 |
+
batch, boosts, target_ok, obs_ok, label_ok, hands_ok, episodes = [], [], 0, 0, 0, 0, set()
|
| 229 |
+
not_delayed = 0
|
| 230 |
+
for i in index:
|
| 231 |
+
sample = dataset[i]
|
| 232 |
+
info = dataset.last_augmentation
|
| 233 |
+
base = WholeBodyLeRobotDataset.__getitem__(dataset, i)
|
| 234 |
+
p = info['visual']
|
| 235 |
+
boosts.append(p['boost'])
|
| 236 |
+
episodes.add(sample['episode_index'])
|
| 237 |
+
views = torch.stack((((base['video'].permute(1, 0, 2, 3) + 1) * .5).clamp(0, 1)).chunk(2, dim=-1), dim=1)
|
| 238 |
+
clean = A.apply(views, p, torch.tensor([False, False]))
|
| 239 |
+
got = torch.stack((((sample['video'].permute(1, 0, 2, 3) + 1) * .5)).chunk(2, dim=-1), dim=1)
|
| 240 |
+
target_ok += bool(torch.allclose(got[1], clean[1], atol=1e-4))
|
| 241 |
+
sensor = any(active(p, k) for k in SENSOR)
|
| 242 |
+
d0 = (got[0] - clean[0]).abs().max().item()
|
| 243 |
+
obs_ok += (d0 > 1e-6) if sensor else (d0 < 1e-4)
|
| 244 |
+
label_ok += bool(torch.equal(sample['action'], base['action']))
|
| 245 |
+
if info['numeric']['delay_recorded_frames'] == 0:
|
| 246 |
+
not_delayed += 1
|
| 247 |
+
hands_ok += bool(torch.allclose(sample['proprio'][:, 29:44], base['proprio'][:, 29:44])
|
| 248 |
+
and not torch.allclose(sample['proprio'][:, :29], base['proprio'][:, :29]))
|
| 249 |
+
batch.append(sample)
|
| 250 |
+
n = len(index)
|
| 251 |
+
check(target_ok == n, f't+16 target = colour/geometry/light/WB only, no sensor effects ({target_ok}/{n})')
|
| 252 |
+
check(obs_ok == n, f'observation frame gets the sensor/optics effects when drawn ({obs_ok}/{n})')
|
| 253 |
+
check(label_ok == n, f'action labels identical to the un-augmented loader ({label_ok}/{n})')
|
| 254 |
+
check(hands_ok == not_delayed, f'undelayed samples: hands/gravity untouched, body joints perturbed ({hands_ok}/{not_delayed})')
|
| 255 |
+
check(max(episodes) < 100, f'samples only from episodes < 100 (max {max(episodes)})')
|
| 256 |
+
check(len(set(boosts)) >= min(8, n // 3), f'boosted effects seen: {sorted(set(boosts))}')
|
| 257 |
+
check(all(s['proprio'].shape[-1] == 44 for s in batch), 'every sample proprio is 44-D (no velocity)')
|
| 258 |
+
b = default_collate(batch[:8])
|
| 259 |
+
k = len(batch[:8])
|
| 260 |
+
shapes = {key: tuple(b[key].shape) for key in ('video', 'action', 'past_action', 'proprio', 'text_hidden_states')}
|
| 261 |
+
check(shapes == {'video': (k, 3, 2, 224, 448), 'action': (k, 16, 76), 'past_action': (k, 16, 76),
|
| 262 |
+
'proprio': (k, 16, 44), 'text_hidden_states': (k, 128, 7680)}, f'batch shapes {shapes}')
|
| 263 |
+
check(bool(torch.isfinite(b['proprio']).all()) and b['video'].min() >= -1 and b['video'].max() <= 1,
|
| 264 |
+
'finite proprio, video in [-1,1]')
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def check_task_h2(A, samples):
|
| 268 |
+
"""A18 / H2: 75-D action, 78-D proprio (75 joints + gravity 3, NO velocity), cameras head + chest."""
|
| 269 |
+
import yaml
|
| 270 |
+
from hydra.utils import instantiate
|
| 271 |
+
from omegaconf import OmegaConf
|
| 272 |
+
from torch.utils.data import default_collate
|
| 273 |
+
from imagewam.datasets.h2_pour import CAMERAS, H2PourDataset
|
| 274 |
+
import imagewam.h2_pour_delta as model_module
|
| 275 |
+
R = ROOT / 'a18'
|
| 276 |
+
print(f'[3] task a18 (H2): {R}')
|
| 277 |
+
check((R / 'prepared.ok').is_file(), 'prepared.ok present')
|
| 278 |
+
receipt = json.loads((R / 'data-verified.json').read_text())
|
| 279 |
+
u, frames = receipt['updates_per_epoch'], receipt['frames']
|
| 280 |
+
check(frames == 72752 and u == 285 and receipt['train_episode_ids'] == list(range(100)),
|
| 281 |
+
f'A18: 100 episodes, {frames} frames, {u} updates/epoch')
|
| 282 |
+
check(not list(R.rglob('*velocity*')), 'no velocity cache / statistics files')
|
| 283 |
+
cfg = yaml.safe_load((R / 'run/production.yaml').read_text())
|
| 284 |
+
d, m = cfg['data']['train'], cfg['model']
|
| 285 |
+
left = velocity_entries(cfg)
|
| 286 |
+
check(not left and not d['proprio_augmentation'].get('velocity_channels'),
|
| 287 |
+
f'config has no joint-velocity setting ({left or "none"})')
|
| 288 |
+
saves = [e * u for e in (10, 20, 30, 40)]
|
| 289 |
+
check(cfg['num_epochs'] == 40 and cfg['max_steps'] == cfg['lr_scheduler_total_steps'] == 40 * u
|
| 290 |
+
and cfg['warmup_steps'] == 2 * u and cfg['save_steps'] == saves and cfg['save_every'] == 0
|
| 291 |
+
and cfg['keep_latest_weights'] == 4 and cfg['keep_latest_state_only'] is True and cfg['save_at_end'] is True,
|
| 292 |
+
f'schedule: max_steps {40 * u}, warmup {2 * u}, saves {saves}, newest state only')
|
| 293 |
+
pa = d['proprio_augmentation']
|
| 294 |
+
check(d['_target_'].endswith('H2HBAugV2Dataset') and d['image_augmentation']['version'] == A.VERSION
|
| 295 |
+
and pa['affected_angle_channels'] == list(range(31)) and pa['control_hz'] == pa['recording_hz'] == 30,
|
| 296 |
+
'H2 dataset class, aug v2, bias/noise on state[0:31], 30 Hz delay conversion')
|
| 297 |
+
check('h2_pour_delta' in m['_target_'] and m['proprio_dim'] == m['action_state_dim'] == 78
|
| 298 |
+
and m['expected_action_dim'] == 75, f'model: original H2 h2_pour_delta, state 78 (no velocity) / action 75')
|
| 299 |
+
sig = __import__('inspect').signature(model_module.create_imagewam_flux2_wholebody_delta)
|
| 300 |
+
check(sig.parameters['action_state_dim'].default == 78, 'h2_pour_delta factory default state 78')
|
| 301 |
+
check(list(CAMERAS) == ['observation.images.head', 'observation.images.chest'], f'cameras {list(CAMERAS)}')
|
| 302 |
+
|
| 303 |
+
dataset = instantiate(OmegaConf.load(R / 'run/production.yaml').data.train)
|
| 304 |
+
check(len(dataset) == frames, f'dataset {len(dataset)} samples')
|
| 305 |
+
rng = np.random.default_rng(99)
|
| 306 |
+
index = sorted(set(rng.integers(0, len(dataset), samples).tolist()) | {0})
|
| 307 |
+
batch, boosts, target_ok, obs_ok, label_ok, hands_ok, not_delayed = [], [], 0, 0, 0, 0, 0
|
| 308 |
+
for i in index:
|
| 309 |
+
sample = dataset[i]
|
| 310 |
+
info = dataset.last_augmentation
|
| 311 |
+
base = H2PourDataset.__getitem__(dataset, i) # parent path: augmentation disabled, 78-D state
|
| 312 |
+
p = info['visual']
|
| 313 |
+
boosts.append(p['boost'])
|
| 314 |
+
views = torch.stack((((base['video'].permute(1, 0, 2, 3) + 1) * .5).clamp(0, 1)).chunk(2, dim=-1), dim=1)
|
| 315 |
+
clean = A.apply(views, p, torch.tensor([False, False]))
|
| 316 |
+
got = torch.stack((((sample['video'].permute(1, 0, 2, 3) + 1) * .5)).chunk(2, dim=-1), dim=1)
|
| 317 |
+
target_ok += bool(torch.allclose(got[1], clean[1], atol=1e-4))
|
| 318 |
+
d0 = (got[0] - clean[0]).abs().max().item()
|
| 319 |
+
obs_ok += (d0 > 1e-6) if any(active(p, k) for k in SENSOR) else (d0 < 1e-4)
|
| 320 |
+
label_ok += bool(torch.equal(sample['action'], base['action']))
|
| 321 |
+
numeric = info['numeric']
|
| 322 |
+
if numeric['delay_recorded_frames'] == 0:
|
| 323 |
+
not_delayed += 1
|
| 324 |
+
hands_ok += bool(torch.allclose(sample['proprio'][:, 31:78], base['proprio'][:, 31:78])
|
| 325 |
+
and not torch.allclose(sample['proprio'][:, :31], base['proprio'][:, :31]))
|
| 326 |
+
batch.append(sample)
|
| 327 |
+
n = len(index)
|
| 328 |
+
check(target_ok == n, f't+16 target = colour/geometry/light/WB only ({target_ok}/{n})')
|
| 329 |
+
check(obs_ok == n, f'observation frame gets the sensor/optics effects when drawn ({obs_ok}/{n})')
|
| 330 |
+
check(label_ok == n, f'action labels identical to the un-augmented H2 loader ({label_ok}/{n})')
|
| 331 |
+
check(all(s['proprio'].shape[-1] == 78 for s in batch), 'every sample proprio is 78-D (no velocity)')
|
| 332 |
+
check(hands_ok == not_delayed, f'undelayed: hands/gravity untouched, 31 body joints perturbed ({hands_ok}/{not_delayed})')
|
| 333 |
+
check(len(set(boosts)) >= min(8, n // 3), f'boosted effects seen: {sorted(set(boosts))}')
|
| 334 |
+
b = default_collate(batch[:8])
|
| 335 |
+
k = len(batch[:8])
|
| 336 |
+
shapes = {key: tuple(b[key].shape) for key in ('video', 'action', 'past_action', 'proprio', 'proprio_dim_is_pad')}
|
| 337 |
+
check(shapes == {'video': (k, 3, 2, 224, 448), 'action': (k, 16, 75), 'past_action': (k, 16, 75),
|
| 338 |
+
'proprio': (k, 16, 78), 'proprio_dim_is_pad': (k, 78)}, f'batch shapes {shapes}')
|
| 339 |
+
check(bool(torch.isfinite(b['proprio']).all()) and b['video'].min() >= -1 and b['video'].max() <= 1,
|
| 340 |
+
'finite proprio, video in [-1,1]')
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def main():
|
| 344 |
+
parser = argparse.ArgumentParser()
|
| 345 |
+
parser.add_argument('tasks', nargs='*', default=['b6', 'a3', 'a7', 'a18'])
|
| 346 |
+
parser.add_argument('--samples', type=int, default=32)
|
| 347 |
+
parser.add_argument('--aug-only', action='store_true')
|
| 348 |
+
parser.add_argument('--skip-aug', action='store_true', help=argparse.SUPPRESS)
|
| 349 |
+
args = parser.parse_args()
|
| 350 |
+
torch.set_num_threads(4)
|
| 351 |
+
g1 = [t for t in args.tasks if t != 'a18']
|
| 352 |
+
h2_rc = 0
|
| 353 |
+
if 'a18' in args.tasks and g1 and not args.aug_only:
|
| 354 |
+
# The H2 code tree has its own imagewam package: check A18 in a separate interpreter.
|
| 355 |
+
import subprocess
|
| 356 |
+
h2_rc = subprocess.call([sys.executable, __file__, 'a18', '--samples', str(args.samples), '--skip-aug'])
|
| 357 |
+
h2_only = args.tasks == ['a18'] and not args.aug_only
|
| 358 |
+
if args.aug_only:
|
| 359 |
+
sys.path.insert(0, str(G / 'hb_groot'))
|
| 360 |
+
import hb_aug_v2 as A
|
| 361 |
+
else:
|
| 362 |
+
sys.path.insert(0, str((CODE_H2 if h2_only else CODE) / 'src'))
|
| 363 |
+
from imagewam.datasets import hb_aug_v2 as A
|
| 364 |
+
print('hb_aug_v2 from', A.__file__)
|
| 365 |
+
if not args.skip_aug:
|
| 366 |
+
check_parameters(A)
|
| 367 |
+
check_apply(A)
|
| 368 |
+
if not args.aug_only:
|
| 369 |
+
if h2_only:
|
| 370 |
+
check_task_h2(A, args.samples)
|
| 371 |
+
else:
|
| 372 |
+
for task in g1:
|
| 373 |
+
check_task(task, A, args.samples)
|
| 374 |
+
if h2_rc:
|
| 375 |
+
FAILURES.append('a18 sub-check failed (see its output above)')
|
| 376 |
+
print('VERIFY_FAIL %d' % len(FAILURES) if FAILURES else 'VERIFY_OK', flush=True)
|
| 377 |
+
return 1 if FAILURES else 0
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
if __name__ == '__main__':
|
| 381 |
+
sys.exit(main())
|
code/hb_deltaflow/wholebody_hbaug.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ImageWAM Delta-flow loader with Humanoid Benchmark augmentation spec v2 (project CLAUDE.md), proprio aligned with
|
| 2 |
+
the GR00T A20 recipe (2026-11-12).
|
| 3 |
+
|
| 4 |
+
Installed by prepare_deltaflow.py as imagewam/datasets/wholebody_hbaug.py, next to an unchanged copy of
|
| 5 |
+
$G/hb_groot/hb_aug_v2.py (the implementation shared by GR00T / pi0.5 / Psi-Zero / DiT4DiT).
|
| 6 |
+
|
| 7 |
+
Base class: WholeBodyRobustLeRobotDataset (wholebody_augmented_v3), the loader of the reference run
|
| 8 |
+
HF humanoid-benchmark/b6-imagewam-deltaflow76 real30-from-sim60-20261108 (proprio_dim 44, velocity_channels []).
|
| 9 |
+
NO joint velocity (user rule 2026-11-10): the model input is exactly 44-D = body joints 29 + measured hands 12 +
|
| 10 |
+
projected gravity 3. Any velocity argument or any proprio width other than 44 is an error.
|
| 11 |
+
|
| 12 |
+
Visual: hb_aug_v2.augment on [T=2, V=2, 3, 224, 224]: one of the 15 effects uniformly boosted to 1.5x per sample;
|
| 13 |
+
colour shared over both cameras and both times; rotation/crop, light patches and single-channel white balance drawn
|
| 14 |
+
per camera and applied identically to the observation (t) and the future target (t+16); noise / gaussian blur / JPEG /
|
| 15 |
+
motion blur / sharpen / defocus only on the observation frame.
|
| 16 |
+
Proprio (2026-11-12, user: align every model with GR00T's A20 basket-transfer training, hb_groot/hb_launch.py):
|
| 17 |
+
hb_aug_v2.perturb_joints on the 29 body joints of the CURRENT frame, i.e. a fresh uniform +-2 deg bias per joint and
|
| 18 |
+
per sample plus N(0, 0.25 deg) noise clipped at 3 sigma; NO proprio delay, NO action-history drop (the parent class's
|
| 19 |
+
per-episode bias, 15 % delay and 15 % history drop are replaced). Labels (action, measured hands) are never modified.
|
| 20 |
+
"""
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import math
|
| 24 |
+
|
| 25 |
+
import numpy as np
|
| 26 |
+
import torch
|
| 27 |
+
|
| 28 |
+
from . import hb_aug_v2
|
| 29 |
+
from .wholebody_augmented_v3 import WholeBodyRobustLeRobotDataset
|
| 30 |
+
|
| 31 |
+
SPEC_VERSION = "hb-aug-v2-20261109"
|
| 32 |
+
PROPRIO_DIM = 44 # body 29 + hands 12 + gravity 3; NO velocity
|
| 33 |
+
BODY = 29
|
| 34 |
+
PROPRIO_SPEC = {"enabled": True, "joint_bias_degrees": 2.0, "joint_noise_std_degrees": 0.25,
|
| 35 |
+
"delay_probability": 0.0, "max_delay_control_cycles": 0, "recording_hz": 30,
|
| 36 |
+
"history_drop_probability": 0.0, "bias_scope": "per_sample",
|
| 37 |
+
"implementation": "hb_aug_v2.perturb_joints"}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class WholeBodyHBAugV2LeRobotDataset(WholeBodyRobustLeRobotDataset):
|
| 41 |
+
def __init__(self, *, image_augmentation, proprio_augmentation, **kwargs):
|
| 42 |
+
velocity_args = sorted(k for k in kwargs if "velocity" in k)
|
| 43 |
+
if velocity_args:
|
| 44 |
+
raise ValueError(f"Joint velocity is not allowed in the model input (got {velocity_args})")
|
| 45 |
+
image_policy = dict(image_augmentation)
|
| 46 |
+
proprio_policy = dict(proprio_augmentation)
|
| 47 |
+
if hb_aug_v2.VERSION != SPEC_VERSION or image_policy.get("version") != SPEC_VERSION:
|
| 48 |
+
raise ValueError(f"Expected augmentation {SPEC_VERSION}, got module {hb_aug_v2.VERSION} / "
|
| 49 |
+
f"config {image_policy.get('version')}")
|
| 50 |
+
if image_policy.get("enabled") is not True or "seed" not in image_policy:
|
| 51 |
+
raise ValueError("Augmentation v2 must be enabled and seeded")
|
| 52 |
+
for key, value in PROPRIO_SPEC.items():
|
| 53 |
+
if proprio_policy.get(key) != value:
|
| 54 |
+
raise ValueError(f"proprio_augmentation.{key}={proprio_policy.get(key)!r}, aligned spec needs {value!r}")
|
| 55 |
+
if list(proprio_policy.get("affected_angle_channels", [])) != list(range(BODY)):
|
| 56 |
+
raise ValueError("Bias/noise must hit exactly the 29 body joints")
|
| 57 |
+
if list(proprio_policy.get("velocity_channels") or []):
|
| 58 |
+
raise ValueError("velocity_channels must be empty (no joint velocity)")
|
| 59 |
+
super().__init__(image_augmentation=image_policy, proprio_augmentation=proprio_policy, **kwargs)
|
| 60 |
+
|
| 61 |
+
def __getitem__(self, index):
|
| 62 |
+
sample = super().__getitem__(index)
|
| 63 |
+
width = int(sample["proprio"].shape[-1])
|
| 64 |
+
pad = sample.get("proprio_dim_is_pad")
|
| 65 |
+
if width != PROPRIO_DIM or (pad is not None and int(pad.shape[-1]) != PROPRIO_DIM):
|
| 66 |
+
raise ValueError(f"proprio must be {PROPRIO_DIM}-D (no velocity), got {width}")
|
| 67 |
+
return sample
|
| 68 |
+
|
| 69 |
+
def augment_numeric(self, sample, rng):
|
| 70 |
+
episode = int(sample["episode_index"])
|
| 71 |
+
anchor = int(sample["frame_index"])
|
| 72 |
+
states, _ = self.episodes[episode]
|
| 73 |
+
state = states[anchor].copy() # no delay: always the current frame
|
| 74 |
+
raw = state[:BODY].astype(np.float64)
|
| 75 |
+
state[:BODY] = hb_aug_v2.perturb_joints(state[:BODY], rng) # per-sample bias +-2 deg + noise 0.25 deg
|
| 76 |
+
current = self.normalizer.forward(state, "state")
|
| 77 |
+
sample["proprio"] = torch.from_numpy(np.repeat(current[None], self.action_horizon, axis=0)).to(
|
| 78 |
+
dtype=sample["proprio"].dtype)
|
| 79 |
+
past = sample["past_action"].clone()
|
| 80 |
+
pad = sample["past_action_is_pad"].clone()
|
| 81 |
+
past[pad] = 0 # unchanged history, no drop
|
| 82 |
+
sample["past_action"] = past
|
| 83 |
+
sample["past_action_is_pad"] = pad
|
| 84 |
+
return dict(delay_recorded_frames=0, history_drop_rows=[],
|
| 85 |
+
joint_perturbation_degrees=(np.degrees(state[:BODY].astype(np.float64) - raw)).tolist())
|
| 86 |
+
|
| 87 |
+
def augment_video(self, video, rng):
|
| 88 |
+
# video: [3, T, 224, 2*224] in [-1, 1]; T=2 = (anchor t, future target t+16); width = ego_view | chest.
|
| 89 |
+
if (video.ndim != 4 or video.shape[0] != 3 or video.shape[1] != 2
|
| 90 |
+
or video.shape[-1] != 2 * video.shape[-2]):
|
| 91 |
+
raise ValueError(f"Expected [3,2,H,2H] ego|chest video, got {tuple(video.shape)}")
|
| 92 |
+
x = ((video.permute(1, 0, 2, 3) + 1.0) * 0.5).clamp(0, 1)
|
| 93 |
+
views = torch.stack(x.chunk(2, dim=-1), dim=1) # [T, V=2, 3, H, W]
|
| 94 |
+
observation = torch.ones(views.shape[0], dtype=torch.bool)
|
| 95 |
+
observation[-1] = False # t+16 is the supervised image target
|
| 96 |
+
out, params = hb_aug_v2.augment(views, rng, obs_mask=observation)
|
| 97 |
+
joined = torch.cat(list(out.unbind(1)), dim=-1) # [T, 3, H, 2W]
|
| 98 |
+
return (joined * 2.0 - 1.0).permute(1, 0, 2, 3).contiguous().to(video.dtype), params
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
assert math.isclose(PROPRIO_SPEC["joint_bias_degrees"], 2.0)
|