Nadeem-Shoukath115113's picture
download
raw
25.9 kB
"""Generate progress.npy and reward.npy for each episode based on labels.
Slope rules (relative to a base slope s):
- progress_fast: +2s
- progress_slow: +1s
- adjust: 0
- mistake: -1s
Base slope s is solved so that integrating the piecewise-linear curve
from t=0 to t=T yields exactly `final_progress`.
If catastrophic failure is marked, progress after that frame is set to 0.
reward.npy = np.diff(progress, prepend=0) (per-step delta).
--mode clipped (opt-in) fixes two cases the default linear solve cannot represent:
- inverted: when mistake time outweighs progress time, sum(multiplier * dt) < 0, so reaching
a positive final_progress needs a NEGATIVE base slope, which flips every segment -- progress
falls during "progress" segments and rises during "mistake" segments.
- flat: when final_progress == start_progress the base slope is 0, so any progress before a
mistake is erased.
Clipped mode instead simulates p(t) = clip(p(t-1) + s * multiplier * dt, 0, 1) with s >= 0, and
mistakes scaled by a severity k >= 0: a mistake pulls progress down but never below 0 and never
changes sign. Mistake-free episodes get the smallest s that ends at final_progress. Every episode
with mistakes (and any with final == start) shares one s -- the median of the mistake-free
episodes in the same results file -- and solves k instead, so a "fast" second means the same
progress everywhere and only mistake severity varies. Labels no curve can satisfy (e.g. only
"mistake" segments but final 1.0) keep the linear result and are flagged. Catastrophic keeps its
meaning: the curve reaches final_progress at the marked frame (or the first "catastrophic"
segment) and is 0 afterwards.
--mode fuse (opt-in, needs --pred-root) combines the clipped curve with a reward model's
prediction (<pred-root>/episode_XXXXXX/pred.npy, e.g. from tools/make_sarm2_video.py):
1. The prediction is forced onto the label. At each segment boundary it keeps the model's
value (1 s moving average) unless the label rules it out -- progress segments must rise,
mistakes fall (by at least 0.03 where feasible), adjust stays flat, everything in [0, 1],
and final_progress must stay reachable -- in which case it takes the nearest allowed value.
Inside a segment it follows the model's timing only as far as the model itself moved in the
labelled direction; the rest of the change is a straight line.
2. progress = pred_weight * (1) + (1 - pred_weight) * clipped curve. Both obey the label, so the
mix does too and ends exactly at final_progress.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import numpy as np
SLOPE_MULT = {
"progress_fast": 2.0,
"progress_slow": 1.0,
"adjust": 0.0,
"mistake": -1.0,
"catastrophic": 0.0,
}
def load_labels(results_path: Path) -> list[dict]:
"""Load all labels from a JSONL results file. Keep last label per episode."""
labels = {}
with results_path.open() as f:
for line in f:
row = json.loads(line)
# Later entries override earlier (re-labels)
labels[row["video_path"]] = row
return list(labels.values())
def video_path_to_episode_dir(video_path: str) -> Path:
"""Extract episode directory from video path."""
return Path(video_path).parent
def _multipliers_and_dt(timestamp: np.ndarray, segments: list[dict]) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Relative time, per-step slope multiplier and per-step dt for an episode."""
n = len(timestamp)
t = timestamp - timestamp[0] # relative time from start
# Map each timestep to its segment's slope multiplier
multipliers = np.zeros(n, dtype=np.float64)
for seg in segments:
seg_start = seg["start"]
seg_end = seg["end"]
label = seg["label"]
m = SLOPE_MULT.get(label, 0.0)
mask = (t >= seg_start) & (t < seg_end)
multipliers[mask] = m
# Include the very last frame if it falls on a segment boundary
if segments:
last_label = segments[-1]["label"]
multipliers[t >= segments[-1]["start"]] = SLOPE_MULT.get(last_label, 0.0)
# Compute dt between consecutive timesteps
dt = np.diff(t, prepend=0)
dt[0] = 0 # first step has no delta
return t, multipliers, dt
def compute_progress(
timestamp: np.ndarray,
segments: list[dict],
final_progress: float,
catastrophic_frame: float | None,
start_progress: float = 0.0,
) -> np.ndarray:
"""Compute progress curve aligned to timestamp array.
Curve starts at ``start_progress`` and reaches ``final_progress`` at the end
(or at ``catastrophic_frame`` if set, after which progress is zeroed).
"""
n = len(timestamp)
t, multipliers, dt = _multipliers_and_dt(timestamp, segments)
if catastrophic_frame is not None:
# Only count weighted sum up to catastrophic frame
cat_mask = t < catastrophic_frame
weighted_sum = np.sum(multipliers[cat_mask] * dt[cat_mask])
else:
weighted_sum = np.sum(multipliers * dt)
delta = final_progress - start_progress
if abs(weighted_sum) < 1e-9:
if abs(delta) < 1e-9:
progress = np.full(n, start_progress, dtype=np.float64)
else:
# No nonzero slopes but start != final — linear interpolation
progress = np.linspace(start_progress, final_progress, n)
else:
base_slope = delta / weighted_sum
# Integrate: progress[i] = start + sum of base_slope * m[j] * dt[j] for j<=i
increments = base_slope * multipliers * dt
progress = start_progress + np.cumsum(increments)
# Clamp to [0, 1] for non-catastrophic
if catastrophic_frame is None:
progress = np.clip(progress, 0.0, 1.0)
else:
# Zero out after catastrophic frame
cat_mask = t >= catastrophic_frame
progress[cat_mask] = 0.0
# Clip pre-catastrophic portion
progress = np.clip(progress, 0.0, 1.0)
return progress
def effective_catastrophic_frame(label: dict) -> float | None:
"""The marked catastrophic frame, else the start of the first "catastrophic" segment.
The UI's mark button sets catastrophic_frame; picking "catastrophic" from the segment dropdown
does not, although it means the same thing.
"""
if label.get("catastrophic_frame") is not None:
return label["catastrophic_frame"]
starts = [s["start"] for s in label["segments"] if s["label"] == "catastrophic"]
return min(starts) if starts else None
K_MAX = 10.0 # ceiling on mistake severity k in the clipped solve
def _simulate_clipped(
multipliers: np.ndarray, dt: np.ndarray, speed: float, start: float, mistake_scale: float = 1.0
) -> np.ndarray:
"""p[i] = clip(p[i-1] + speed * m[i] * dt[i], 0, 1), with negative m scaled by mistake_scale."""
m = np.where(multipliers < 0, multipliers * mistake_scale, multipliers)
steps = speed * m * dt
progress = np.empty(len(steps), dtype=np.float64)
cur = start
for i, step in enumerate(steps):
cur = min(max(cur + step, 0.0), 1.0)
progress[i] = cur
return progress
def _bisect(fn, lo: float, hi: float, iters: int = 60) -> float:
"""Boundary in [lo, hi] where fn flips from False to True (fn(hi) assumed True)."""
for _ in range(iters):
mid = 0.5 * (lo + hi)
if fn(mid):
hi = mid
else:
lo = mid
return hi
def has_mistakes(label: dict) -> bool:
return any(s["label"] == "mistake" for s in label["segments"])
def solve_clipped(
timestamp: np.ndarray,
label: dict,
fallback_speed: float | None = None,
force_fallback: bool = False,
mistake_gain: float = 1.0,
) -> tuple[float, float, float, str]:
"""(speed, mistake_scale, abs end error, how) for the clipped curve.
1. If final != start (and not force_fallback), find the smallest speed that ends at final
with mistakes undoing progress at the same rate (mistake_scale 1): "exact".
2. Otherwise the speed is fallback_speed (raised if it cannot reach final) and the mistake
severity is solved. The caller forces this for episodes with mistakes, so one progress
rate is shared across them; per-episode speeds let a short final climb inflate the rate
for the whole episode (e.g. "fully done at 15 s, then undone").
"""
start = label.get("start_progress", 0.0)
final = label["final_progress"]
t, multipliers, dt = _multipliers_and_dt(timestamp, label["segments"])
cat = effective_catastrophic_frame(label)
before = np.nonzero(t < cat)[0] if cat is not None else np.arange(len(t))
end = int(before[-1]) if len(before) else 0 # catastrophic from frame 0: nothing to fit
multipliers, dt = multipliers[: end + 1], dt[: end + 1]
def end_value(speed: float, scale: float = 1.0) -> float:
return _simulate_clipped(multipliers, dt, speed, start, scale)[-1]
grid = np.geomspace(1e-5, 10.0, 80)
if abs(final - start) >= 1e-9 and not (force_fallback and fallback_speed is not None):
sign = 1.0 if final > start else -1.0
for lo, hi in zip(np.concatenate([[0.0], grid[:-1]]), grid):
if sign * (end_value(hi) - final) >= 0:
speed = _bisect(lambda s: sign * (end_value(s) - final) >= 0, lo, hi)
err = abs(end_value(speed) - final)
if err < 1e-3:
return speed, 1.0, err, "exact"
break
if fallback_speed is None:
return 0.0, 1.0, abs(end_value(0.0) - final), "unsolved"
speed = max(fallback_speed, 1e-9)
# Enough speed that, with mistakes disabled, the curve can reach final.
if end_value(speed, 0.0) < final:
hi = next((s for s in grid if s > speed and end_value(s, 0.0) >= final), None)
if hi is None:
return speed, 0.0, abs(end_value(speed, 0.0) - final), "unreachable"
speed = _bisect(lambda s: end_value(s, 0.0) >= final, speed, hi)
# HEADROOM. The line above pins `speed` to the SMALLEST rate that reaches final_progress with
# the mistakes disabled, which leaves nothing for a mistake to give back: any k > 0 then ends
# below final, so the severity solve below is forced to k ~= 0 and the labelled mistake shows
# no dip. On this corpus that is every mistake episode whose final_progress is 1.0 -- the
# human said "ended fully folded" AND "part of it was a mistake", and the solver resolved the
# tension by silently discarding the mistake.
#
# Running the good segments faster than the bare minimum resolves it the other way: the robot
# did progress quickly, and a mistake really did undo some of it. Ease the gain back if even
# the harshest mistake could not bring the ending down to final_progress again.
if mistake_gain > 1.0:
base = speed
if end_value(base * mistake_gain, K_MAX) > final + 1e-9:
mistake_gain = _bisect(
lambda x: end_value(base * x, K_MAX) > final + 1e-9, 1.0, mistake_gain)
speed = base * mistake_gain
# More severe mistakes can only lower the ending: solve the severity that lands on final.
if end_value(speed, K_MAX) > final + 1e-9:
return speed, K_MAX, abs(end_value(speed, K_MAX) - final), "unreachable"
# end_value is non-increasing in k, but it is usually FLAT at `final` over a whole interval of
# k: the curve clips at 1.0, so once the segments after a mistake re-saturate, the ending is
# the same however hard the mistake hit. Every k in that interval satisfies the label equally.
# Taking the smallest -- which a False->True bisect on `end <= final` does, since that test
# already passes at k=0 -- resolves a labelled mistake to NO DIP AT ALL. On this corpus that
# was 9 of 11 mistake episodes: every one whose final_progress is 1.0.
#
# Take the largest k instead: the deepest dip the label still permits. It is self-limiting,
# because k can only grow while the post-mistake segments still have enough climb left to get
# back to final_progress -- a mistake the robot never recovered from stays shallow, and one it
# spent a long time recovering from goes deep.
if end_value(speed, K_MAX) >= final - 1e-9:
scale = K_MAX
else:
scale = _bisect(lambda k: end_value(speed, k) < final - 1e-9, 0.0, K_MAX)
return speed, scale, abs(end_value(speed, scale) - final), "median-speed"
def compute_progress_clipped(
timestamp: np.ndarray,
label: dict,
speed: float,
mistake_scale: float = 1.0,
) -> np.ndarray:
"""Clipped-simulation curve; zero from the (effective) catastrophic frame on."""
t, multipliers, dt = _multipliers_and_dt(timestamp, label["segments"])
progress = _simulate_clipped(multipliers, dt, speed, label.get("start_progress", 0.0), mistake_scale)
cat = effective_catastrophic_frame(label)
if cat is not None:
progress[t >= cat] = 0.0
return progress
def clipped_curves(
episodes: list[tuple[Path, np.ndarray, dict]], mistake_gain: float = 1.0
) -> list[tuple[np.ndarray, str]]:
"""Clipped-mode (progress, note) for every (episode_dir, timestamp, label)."""
# Pass 1: solve every episode the label fully determines, so the rest can borrow the median
# speed of those.
exact = [(solve_clipped(ts, lab), has_mistakes(lab)) for _, ts, lab in episodes]
# The shared progress rate comes from mistake-free episodes, where the label pins it down
# without any assumption about how hard a mistake hits; fall back to all exact solves.
clean = [r[0] for r, mistakes in exact if r[3] == "exact" and not mistakes]
speeds = clean if len(clean) >= 3 else [r[0] for r, _ in exact if r[3] == "exact"]
median_speed = float(np.median(speeds)) if speeds else None
print(f"[clipped] shared speed {median_speed if median_speed is not None else float('nan'):.5f} "
f"from {len(speeds)} {'mistake-free ' if speeds is clean else ''}episodes; used for every "
f"episode with mistakes and wherever final == start")
out = []
for _, timestamp, label in episodes:
gain = mistake_gain if has_mistakes(label) else 1.0
speed, scale, err, how = solve_clipped(
timestamp, label, fallback_speed=median_speed,
force_fallback=has_mistakes(label), mistake_gain=gain)
if err >= 1e-3 and has_mistakes(label):
# The shared speed overshoots final even with the harshest mistakes (progress after
# the last mistake already exceeds it): let this episode's own speed try.
speed, scale, err, how = solve_clipped(
timestamp, label, fallback_speed=median_speed, mistake_gain=gain)
if err < 1e-3:
out.append((compute_progress_clipped(timestamp, label, speed, scale),
f" | {how} speed={speed:.5f} mistake_scale={scale:.3f}"))
else:
# The label contradicts itself (e.g. only "mistake" segments but final 1.0, or one
# "slow" segment with final 0), so no clipped curve satisfies it. Keep the linear
# result rather than invent one, and say so.
out.append((compute_progress(timestamp, label["segments"], label["final_progress"],
label.get("catastrophic_frame"),
start_progress=label.get("start_progress", 0.0)),
f" | LINEAR FALLBACK: label contradicts itself (clipped end off by {err:.3f})"))
return out
UP_LABELS = {"progress_fast", "progress_slow"}
FUSE_MIN_STEP = (0.03, 0.01, 0.0) # smallest rise/fall per segment, relaxed until feasible
def _label_intervals(label: dict, t_end: float) -> list[tuple[float, float, str]]:
"""[0, t_end] cut at every segment boundary, as (start, end, label). Gaps count as adjust and
the last segment runs to the end, as in _multipliers_and_dt."""
segs = sorted(label["segments"], key=lambda s: s["start"])
cuts = sorted({0.0, t_end, *(s["start"] for s in segs), *(s["end"] for s in segs)})
cuts = [c for c in cuts if 0.0 <= c <= t_end]
out = []
for a, b in zip(cuts[:-1], cuts[1:]):
mid = 0.5 * (a + b)
lab = next((s["label"] for s in segs if s["start"] <= mid < s["end"]), None)
if lab is None:
lab = segs[-1]["label"] if segs and mid >= segs[-1]["start"] else "adjust"
out.append((a, b, lab))
return out
def _step_bounds(label: str, min_step: float) -> tuple[float, float]:
if label in UP_LABELS:
return min_step, np.inf
if label == "mistake":
return -np.inf, -min_step
return 0.0, 0.0 # adjust / catastrophic
def _reachable(start: float, final: float, lo: list[float], hi: list[float]) -> bool:
r = (start, start)
for a, b in zip(lo, hi):
r = (max(0.0, r[0] + a), min(1.0, r[1] + b))
if r[0] > r[1] + 1e-12:
return False
return r[0] - 1e-9 <= final <= r[1] + 1e-9
def _follow_knots(target: np.ndarray, start: float, final: float, lo: list[float], hi: list[float]) -> np.ndarray:
"""Left to right, keep each boundary's target unless the step bounds, [0, 1] or reaching
final forbid it; then take the nearest allowed value."""
k = len(lo)
back = [(final, final)] * (k + 1) # values from which final is still reachable
for i in range(k - 1, -1, -1):
back[i] = (max(0.0, back[i + 1][0] - hi[i]), min(1.0, back[i + 1][1] - lo[i]))
knots = [start]
for i in range(k):
a = max(knots[-1] + lo[i], back[i + 1][0])
b = min(knots[-1] + hi[i], back[i + 1][1])
knots.append(float(np.clip(target[i + 1], a, b)))
return np.array(knots)
def _isotonic_increasing(y: np.ndarray) -> np.ndarray:
"""Least-squares non-decreasing fit (pool adjacent violators)."""
vals, wts = [], []
for v in y:
vals.append(float(v))
wts.append(1)
while len(vals) > 1 and vals[-2] > vals[-1]:
w = wts[-2] + wts[-1]
vals[-2] = (vals[-2] * wts[-2] + vals[-1] * wts[-1]) / w
wts[-2] = w
vals.pop()
wts.pop()
return np.repeat(vals, wts)
def fit_prediction_to_label(timestamp: np.ndarray, pred: np.ndarray, label: dict) -> tuple[np.ndarray | None, float | None]:
"""A model's progress prediction forced onto the label (step 1 of --mode fuse).
Returns (curve, min_step), or (None, None) when no curve satisfies the label.
"""
t = timestamp - timestamp[0]
n = len(t)
cat = effective_catastrophic_frame(label)
before = np.nonzero(t < cat)[0] if cat is not None else np.arange(n)
last = int(before[-1]) if len(before) else 0
ivs = _label_intervals(label, float(t[last]))
idx = [0] + [min(int(np.searchsorted(t, b)), last) for _, b, _ in ivs]
if ivs:
idx[-1] = last
keep = [i for i in range(len(ivs)) if idx[i + 1] > idx[i]] # drop sub-frame intervals
ivs = [ivs[i] for i in keep]
idx = [0] + [idx[i + 1] for i in keep]
start, final = label.get("start_progress", 0.0), label["final_progress"]
lo, hi, min_step = [], [], None
for step in FUSE_MIN_STEP:
bounds = [_step_bounds(lab, step) for *_, lab in ivs]
lo, hi = [b[0] for b in bounds], [b[1] for b in bounds]
if _reachable(start, final, lo, hi):
min_step = step
break
if min_step is None:
return None, None
k = max(1, int(round(1.0 / max(float(np.median(np.diff(t))) if n > 1 else 1.0, 1e-6))))
smooth = np.convolve(np.pad(pred, k // 2, mode="edge"), np.ones(k) / k, mode="valid")[:n]
knots = _follow_knots(np.clip(smooth[idx], 0.0, 1.0), start, final, lo, hi) if ivs else np.array([start])
curve = np.zeros(n)
curve[0] = start
for i, (_, _, lab) in enumerate(ivs):
i0, i1 = idx[i], idx[i + 1]
shape = (t[i0:i1 + 1] - t[i0]) / max(t[i1] - t[i0], 1e-9)
if lab in UP_LABELS or lab == "mistake":
sign = 1.0 if lab in UP_LABELS else -1.0
iso = sign * _isotonic_increasing(sign * pred[i0:i1 + 1])
moved, need = sign * (iso[-1] - iso[0]), abs(knots[i + 1] - knots[i])
if moved > 1e-6 and need > 1e-6:
r = min(1.0, moved / need) # the model's timing only as far as it moved itself
shape = r * np.clip(sign * (iso - iso[0]) / moved, 0.0, 1.0) + (1 - r) * shape
curve[i0:i1 + 1] = knots[i] + (knots[i + 1] - knots[i]) * shape
curve[last + 1:] = 0.0
return np.clip(curve, 0.0, 1.0), min_step
def compute_progress_fused(
timestamp: np.ndarray, label: dict, label_curve: np.ndarray, pred: np.ndarray, pred_weight: float = 0.5
) -> tuple[np.ndarray, str]:
"""pred_weight * (prediction forced onto the label) + (1 - pred_weight) * label_curve."""
pred = np.asarray(pred, dtype=np.float64).reshape(-1)
if len(pred) != len(timestamp):
raise SystemExit(f"prediction has {len(pred)} frames, timestamp.npy has {len(timestamp)}")
fitted, min_step = fit_prediction_to_label(timestamp, pred, label)
if fitted is None:
return label_curve, " | fuse: label contradicts itself, label curve only"
return pred_weight * fitted + (1 - pred_weight) * label_curve, f" | fuse w={pred_weight:.2f} min_step={min_step}"
def main():
parser = argparse.ArgumentParser(description="Generate progress.npy and reward.npy from labels")
parser.add_argument("--results", required=True, help="Path to results JSONL file")
parser.add_argument("--dry-run", action="store_true", help="Print what would be written without saving")
parser.add_argument("--mode", choices=["linear", "clipped", "fuse"], default="linear",
help="linear: original single-slope solve. clipped: clipped simulation that "
"cannot invert or erase progress on mistake-heavy episodes. fuse: clipped "
"curve combined with a reward model's prediction (see module doc)")
parser.add_argument("--pred-root", default=None,
help="fuse: dir with episode_XXXXXX/pred.npy (tools/make_sarm2_video.py output)")
parser.add_argument("--mistake-gain", type=float, default=1.0,
help="clipped/fuse: run the progress segments of episodes WITH mistakes "
"this many times faster than the bare minimum that reaches "
"final_progress, which is what gives a labelled mistake room to pull "
"progress DOWN. The curve still ends exactly at final_progress, so a "
"higher value steepens both the rises and the dips. 1.0 reproduces "
"the old behaviour, where mistakes on episodes ending at 1.0 "
"collapsed to no dip at all.")
parser.add_argument("--pred-weight", type=float, default=0.5,
help="fuse: share of the prediction in the result (0 = clipped curve, 1 = prediction "
"forced onto the label)")
args = parser.parse_args()
results_path = Path(args.results)
if not results_path.exists():
raise SystemExit(f"Results file not found: {results_path}")
if args.mode == "fuse" and not args.pred_root:
raise SystemExit("--mode fuse needs --pred-root")
labels = load_labels(results_path)
print(f"Loaded {len(labels)} labeled episodes from {results_path}")
episodes = []
for row in labels:
episode_dir = video_path_to_episode_dir(row["video_path"])
ts_path = episode_dir / "timestamp.npy"
if not ts_path.exists():
print(f"[SKIP] No timestamp.npy in {episode_dir}")
continue
episodes.append((episode_dir, np.load(ts_path), row["label"]))
curves = (clipped_curves(episodes, args.mistake_gain)
if args.mode in ("clipped", "fuse") else None)
for i, (episode_dir, timestamp, label) in enumerate(episodes):
start_progress = label.get("start_progress", 0.0)
final_progress = label["final_progress"]
segments = label["segments"]
catastrophic_frame = label.get("catastrophic_frame")
note = ""
if curves is not None:
progress, note = curves[i]
if args.mode == "fuse":
pred_path = Path(args.pred_root) / episode_dir.name / "pred.npy"
if not pred_path.exists():
raise SystemExit(f"No prediction for {episode_dir.name}: {pred_path}")
progress, fuse_note = compute_progress_fused(
timestamp, label, progress, np.load(pred_path), args.pred_weight)
note += fuse_note
else:
progress = compute_progress(
timestamp,
segments,
final_progress,
catastrophic_frame,
start_progress=start_progress,
)
reward = np.diff(progress, prepend=0.0)
progress_path = episode_dir / "progress.npy"
reward_path = episode_dir / "reward.npy"
if args.dry_run:
print(f"[DRY] {episode_dir.name}: len={len(progress)}, "
f"progress=[{progress[0]:.3f}...{progress[-1]:.3f}] peak={progress.max():.3f}, "
f"final_progress={final_progress}, segments={len(segments)}{note}")
else:
np.save(progress_path, progress)
np.save(reward_path, reward)
print(f"[OK] {episode_dir.name}: saved progress.npy ({len(progress)},) "
f"and reward.npy ({len(reward)},) | final={progress[-1]:.3f}{note}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
25.9 kB
·
Xet hash:
98e23771211f267234622894cfbf6032e8c3a302331acc7f46dba5c0d1ed6866

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.