Blame for PWM_code/reward_model/scripts/gen_pred_flow_pushbox_boxmark.py

#!/usr/bin/env python3
"""Push-Box BOX+MARK point-flow overlay: GT tracks vs label-conditioned 3-PoinTr pred.

Uses the two-object label-conditioned model (push_box_boxmark_150pts_384), tracking
BOTH the cheez-it box (label 0) and the red-cross mark (label 1).

GREEN = GT CoTracker tracks (box+mark).  RED = 3-PoinTr predicted flow from frame-0
GT points + their labels.  Tracks live in 384 space (the videos_small domain they
were built on); we draw on each demo's own input.mp4 (384), upscaled 2x for clarity.

Usage: python gen_pred_flow_pushbox_boxmark.py --ep ep000 [--ep ep005 ...]
"""
import argparse, os, sys
import cv2, imageio
import numpy as np
import torch

POINTR_REPO = "/home/hongyi/PWM/reward_model/3PoinTr_repo"
if POINTR_REPO not in sys.path:
    sys.path.insert(0, POINTR_REPO)
from util.training_utils.arch import TrajectoryModel
from util.geometry_utils.geom_utils import (
    compute_start_sphere_params, sim_transform, sim_transform_inverse_points,
)

CKPT = "/home/hongyi/PWM/reward_model/3pointr_models/push_box_boxmark_150pts_384_flow_backbone_final.pth"
TRACKS_ROOT = "/data0/hongyi/reward_model_data/3pointr_data_push_box_2d_boxmark"   # <ep>/{tracks_labeled.npz,input.mp4}
SQUASH = 384
UPSCALE = 2               # display size = 384*UPSCALE
MAX_POINTS = 150


def infer_horizon(state):
    w = state.get("decoder_only.point_flow_head.weight")
    return None if w is None else int(w.shape[0] // 2)


def load_model(ckpt_path, device):
    ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False)
    cfg = ckpt.get("args", {}) or {}
    state = ckpt.get("flow_backbone_state_dict") or ckpt.get("model_state_dict") or ckpt
    horizon = infer_horizon(state) or int(cfg.get("horizon", 306))
    use_labels = any("point_label_embed" in k for k in state.keys())
    model = TrajectoryModel(
        arch=cfg.get("arch", "flow_conditioned"),
        dim=int(cfg.get("arch_dim", 256)), depth=int(cfg.get("arch_depth", 2)),
        query_depth=int(cfg.get("arch_query_depth", 1)),
        num_heads=int(cfg.get("arch_heads", 4)),
        query_heads=int(cfg.get("arch_query_heads", 1)),
        use_task_conditioning=False, num_tasks=1, horizon=horizon,
        num_queries=int(cfg.get("num_queries", 10)), point_dim=2,
        use_point_labels=use_labels,
    ).to(device)
    missing, unexpected = model.load_state_dict(state, strict=False)
    print(f"[predict] horizon={horizon}  use_point_labels={use_labels}  "
          f"missing={len(missing)} unexpected={len(unexpected)}")
    model.eval()
    return model, horizon


@torch.no_grad()
def predict(model, horizon, x0_px, labels, device):
    x0 = torch.from_numpy(x0_px.astype(np.float32)).to(device).unsqueeze(0)
    scales, center = compute_start_sphere_params(x0)
    inv_scales = 1.0 / scales
    x0n = sim_transform(x0, scales=inv_scales, rotations=None, translations=-center)
    batch = {"x0": x0n, "num_steps": horizon,
             "task_ids": torch.zeros(1, dtype=torch.long, device=device),
             "scales": scales, "center": center,
             "point_labels": torch.from_numpy(labels.astype(np.int64)).to(device).unsqueeze(0)}
    out = model(batch, mode="eval")
    pred = out["pred_pos"]
    L, N = pred.shape[1], pred.shape[2]
    un = sim_transform_inverse_points(pred.reshape(1, L * N, 2),
                                      scales=inv_scales, rotations=None, translations=-center)
    return un.view(L, N, 2).cpu().numpy()


def render(ep, model, horizon, device, out_dir):
    d = f"{TRACKS_ROOT}/tableHF_{ep}"
    z = np.load(f"{d}/tracks_labeled.npz", allow_pickle=True)
    tr = z["tracks"].astype(np.float32)                     # (T,N,2)@384
    lab = z["labels"].astype(np.int64)                      # 0=box 1=mark
    vis = z["vis"] if "vis" in z.files else np.ones(tr.shape[:2], bool)
    N = tr.shape[1]
    if N > MAX_POINTS:
        idx = np.linspace(0, N - 1, MAX_POINTS).astype(int)
        tr, vis, lab = tr[:, idx], vis[:, idx], lab[idx]
        N = tr.shape[1]

    cap = cv2.VideoCapture(f"{d}/input.mp4")
    frames = []
    while True:
        ok, f = cap.read()
        if not ok:
            break
        f = cv2.cvtColor(f, cv2.COLOR_BGR2RGB)
        if UPSCALE != 1:
            f = cv2.resize(f, (SQUASH * UPSCALE, SQUASH * UPSCALE), interpolation=cv2.INTER_LINEAR)
        frames.append(f)
    cap.release()
    Td = min(len(frames), tr.shape[0])
    H, W = frames[0].shape[:2]
    s = float(UPSCALE)                                      # 384 -> display

    pred384 = predict(model, horizon, tr[0], lab, device)
    pred = pred384 * s
    gt = tr * s
    L = pred.shape[0]

    tt = np.linspace(0, L - 1, Td, dtype=np.float32)
    t0 = np.floor(tt).astype(int).clip(0, L - 1); t1 = np.minimum(t0 + 1, L - 1)
    w = (tt - t0)[:, None, None]
    pred_al = (1 - w) * pred[t0] + w * pred[t1]

    out_frames = []
    for ti in range(Td):
        cur = frames[ti].copy()
        for pi in range(N):
            if vis[ti, pi]:
                x, y = gt[ti, pi]; cv2.circle(cur, (int(x), int(y)), 2, (0, 255, 0), -1)
        for pi in range(N):
            x, y = pred_al[ti, pi]; cv2.circle(cur, (int(x), int(y)), 2, (255, 0, 0), -1)
        err = np.linalg.norm(gt[ti] - pred_al[ti], axis=-1)
        m = vis[ti].astype(np.float32); me = float((err * m).sum() / max(m.sum(), 1)) / s
        cv2.putText(cur, f"{ep}  t={ti}  mean err={me:4.1f}px@384", (10, 26),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2)
        cv2.putText(cur, "GT box+mark", (10, H - 34), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
        cv2.putText(cur, "3PoinTr pred", (10, H - 12), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 0, 0), 2)
        out_frames.append(cur)

    os.makedirs(out_dir, exist_ok=True)
    out = f"{out_dir}/pred_flow_{ep}.mp4"
    imageio.mimwrite(out, out_frames, fps=15, macro_block_size=1)
    err = np.linalg.norm(gt[:Td] - pred_al, axis=-1) * vis[:Td]
    me = (err.sum(1) / vis[:Td].sum(1).clip(min=1)) / s
    nb = int((lab == 0).sum()); nm = int((lab == 1).sum())
    print(f"[{ep}] N={N} (box={nb} mark={nm}) L={L} Td={Td}  px err@384 "
          f"first={me[0]:.1f} mid={me[Td//2]:.1f} last={me[-1]:.1f} mean={me.mean():.1f}  -> {out}")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--ep", action="append", default=[])
    ap.add_argument("--ckpt", default=CKPT)
    ap.add_argument("--out_dir", default="/home/hongyi/PWM/external/pred_flow_pushbox_boxmark")
    ap.add_argument("--device", default="cuda:0")
    a = ap.parse_args()
    eps = a.ep or ["ep000"]
    device = torch.device(a.device)
    model, horizon = load_model(a.ckpt, device)
    for ep in eps:
        try:
            render(ep, model, horizon, device, a.out_dir)
        except Exception as e:
            print(f"[{ep}] FAILED: {e}")


if __name__ == "__main__":
    main()