Buckets:

AsianPlayer
/

TriDrive

Files

xet

AsianPlayer/TriDrive / code /data /cut_clips_r2.py

AsianPlayer

17 days ago

download

raw

5.89 kB

"""cwm3/data/cut_clips_r2.py — round-2 TEST annotation clips: NEW manifest cwm2/annotation_human/clips_r2.json + prefix cuts in
cwm2/annotation_human/clips_prefix_r2/ (clips_h5.json and clips_prefix/ are NOT modified). Same recipe as active_select.py:
  full clip  = s15_cockpit/annotate_clips_v2/<idx:05d>_{cab,road}.mp4  (prep_clips_v2.py: window [max(0,t0-5), t0+10] = 15 s, scale 640, crf 27)
  prefix cut = ffmpeg -i <full> -t <anchor_s> -an -c:v libx264 -preset veryfast -crf 23 -movflags +faststart  (cut_prefix); anchor_s = min(5.0, t0)
Per-clip fields as clips_h5.json: idx, uid, t0, t1, dongle, dcam, qcam, clip_id, split, regime, ctrl, anchor_s, prior_label None,
prior_label_protocol None, pilot False, needs_frames (from the draw), double_rate True (all 100), selection {bucket, reason, draw_rank, stratum,
active_score}, clips {prefix_cab, prefix_road, full_cab, full_road}. Duration of every prefix checked <= anchor_s + 0.1 (ffprobe)."""
import argparse, json, subprocess, sys, time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]; EXPLORE = ROOT.parent; sys.path.insert(0, str(ROOT / "common"))
from runmeta import write_run_json
AH = EXPLORE / "cwm2" / "annotation_human"; CLIPS_V2 = EXPLORE / "s15_cockpit" / "annotate_clips_v2"; FFMPEG, FFPROBE = "/usr/bin/ffmpeg", "/usr/bin/ffprobe"; LROOT = Path("/home/USER/BATON")

def cut_prefix(job):
    src, dst, dur = job
    if dst.exists(): return dst, "cached"
    tmp = dst.with_suffix(".tmp.mp4")
    r = subprocess.run([FFMPEG, "-v", "error", "-y", "-i", str(src), "-t", f"{dur:.3f}", "-an", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-movflags", "+faststart", str(tmp)], capture_output=True, text=True)
    if r.returncode != 0: return dst, "fail:" + r.stderr[-200:]
    tmp.rename(dst); return dst, "ok"

def probe_dur(p):
    r = subprocess.run([FFPROBE, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(p)], capture_output=True, text=True)
    try: return float(r.stdout.strip())
    except ValueError: return float("nan")

def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--csv", default=str(ROOT / "results" / "mining" / "round2_draw_v1.csv")); ap.add_argument("--split", default="test"); ap.add_argument("--tag", default="r2"); ap.add_argument("--workers", type=int, default=6); a = ap.parse_args(); t_start = time.time()
    df = pd.read_csv(a.csv); sub = df[df.split == a.split].sort_values("draw_rank"); pre_dir = AH / f"clips_prefix_{a.tag}"; pre_dir.mkdir(exist_ok=True)
    rows, missing_full = [], []
    for r in sub.itertuples():
        idx = int(r.idx); full_cab, full_road = CLIPS_V2 / f"{idx:05d}_cab.mp4", CLIPS_V2 / f"{idx:05d}_road.mp4"
        if not (full_cab.exists() and full_road.exists()): missing_full.append(idx); continue
        p = r.uid.split("__"); leaf = "/".join(p[:3]) + "/" + "__".join(p[3:]); anchor = float(min(5.0, r.t0))
        rows.append(dict(idx=idx, uid=r.uid, t0=float(r.t0), t1=float(r.t0) + 10.0, dongle=r.dongle, dcam=str(LROOT / leaf / "dcamera.mp4"), qcam=str(LROOT / leaf / "qcamera.mp4"), clip_id=r.clip_id, split=r.split, regime=r.regime, ctrl=r.ctrl,
                         anchor_s=anchor, prior_label=None, prior_label_protocol=None, pilot=False, needs_frames=bool(r.needs_frames), double_rate=True,
                         selection=dict(bucket="round2_test", reason="round-2 active draw (results/mining/round2_draw_v1.csv)", draw_rank=int(r.draw_rank), stratum=r.stratum, quota_stratum=r.quota_stratum, active_score=float(r.active_score), kin_source=r.kin_source),
                         clips=dict(prefix_cab=f"clips_prefix_{a.tag}/{idx:05d}_cab.mp4", prefix_road=f"clips_prefix_{a.tag}/{idx:05d}_road.mp4", full_cab=str(full_cab), full_road=str(full_road))))
    jobs = [(Path(r["clips"][f"full_{cam}"]), pre_dir / f"{r['idx']:05d}_{cam}.mp4", r["anchor_s"]) for r in rows for cam in ("cab", "road")]
    stats = {}; fails = []
    with ThreadPoolExecutor(max_workers=a.workers) as ex:
        for dst, st in ex.map(cut_prefix, jobs):
            stats[st.split(":")[0]] = stats.get(st.split(":")[0], 0) + 1
            if st.startswith("fail"): fails.append((dst.name, st))
    bad_dur = []
    for r in rows:
        for cam in ("cab", "road"):
            d = probe_dur(pre_dir / f"{r['idx']:05d}_{cam}.mp4")
            if not (d <= r["anchor_s"] + 0.1): bad_dur.append((r["idx"], cam, d))
    man = dict(generated=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), source_draw=str(Path(a.csv)), split=a.split, schema_version="h5", n=len(rows), anchor_default=5.0, double_rate_all=True,
               note="round-2 TEST manifest for the H5 server; split / prior_label / lineage never sent to the client; full clips = s15 annotate_clips_v2 (15 s, [max(0,t0-5), t0+10]); prefix physically truncated at anchor_s (libx264 crf 23); CAN evidence dirs NOT materialised here (can/<idx>.*.json to be produced by active_select.materialise_can if the server needs them)",
               clips=rows, missing_full_clips=missing_full)
    out = AH / f"clips_{a.tag}.json"; assert not out.exists(), f"{out} exists — refusing to overwrite"; json.dump(man, open(out, "w"), indent=1)
    summ = dict(split=a.split, n_clips=len(rows), missing_full_clips=missing_full, prefix_cuts=stats, cut_failures=fails, duration_violations=bad_dur, distinct_leaves=len({r["dcam"] for r in rows}), double_rate=sum(r["double_rate"] for r in rows), needs_frames=sum(r["needs_frames"] for r in rows), manifest=str(out), prefix_dir=str(pre_dir))
    write_run_json(ROOT / "results", f"cut_clips_{a.tag}", [Path(__file__)], [Path(a.csv)], None, t_start, extra={"summary": summ}, input_listing=False)
    print("CUT_DONE " + json.dumps(summ)); sys.exit(1 if (fails or bad_dur or missing_full) else 0)

if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 5.89 kB
Xet hash:: 922bcc3e00cb6ce551650d49c5369be78d8c416699f2886370b1d70e803b05a9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	"""cwm3/data/cut_clips_r2.py — round-2 TEST annotation clips: NEW manifest cwm2/annotation_human/clips_r2.json + prefix cuts in
	cwm2/annotation_human/clips_prefix_r2/ (clips_h5.json and clips_prefix/ are NOT modified). Same recipe as active_select.py:
	full clip = s15_cockpit/annotate_clips_v2/<idx:05d>_{cab,road}.mp4 (prep_clips_v2.py: window [max(0,t0-5), t0+10] = 15 s, scale 640, crf 27)
	prefix cut = ffmpeg -i <full> -t <anchor_s> -an -c:v libx264 -preset veryfast -crf 23 -movflags +faststart (cut_prefix); anchor_s = min(5.0, t0)
	Per-clip fields as clips_h5.json: idx, uid, t0, t1, dongle, dcam, qcam, clip_id, split, regime, ctrl, anchor_s, prior_label None,
	prior_label_protocol None, pilot False, needs_frames (from the draw), double_rate True (all 100), selection {bucket, reason, draw_rank, stratum,
	active_score}, clips {prefix_cab, prefix_road, full_cab, full_road}. Duration of every prefix checked <= anchor_s + 0.1 (ffprobe)."""
	import argparse, json, subprocess, sys, time
	from concurrent.futures import ThreadPoolExecutor
	from pathlib import Path
	import pandas as pd
	ROOT = Path(__file__).resolve().parents[1]; EXPLORE = ROOT.parent; sys.path.insert(0, str(ROOT / "common"))
	from runmeta import write_run_json
	AH = EXPLORE / "cwm2" / "annotation_human"; CLIPS_V2 = EXPLORE / "s15_cockpit" / "annotate_clips_v2"; FFMPEG, FFPROBE = "/usr/bin/ffmpeg", "/usr/bin/ffprobe"; LROOT = Path("/home/USER/BATON")

	def cut_prefix(job):
	src, dst, dur = job
	if dst.exists(): return dst, "cached"
	tmp = dst.with_suffix(".tmp.mp4")
	r = subprocess.run([FFMPEG, "-v", "error", "-y", "-i", str(src), "-t", f"{dur:.3f}", "-an", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-movflags", "+faststart", str(tmp)], capture_output=True, text=True)
	if r.returncode != 0: return dst, "fail:" + r.stderr[-200:]
	tmp.rename(dst); return dst, "ok"

	def probe_dur(p):
	r = subprocess.run([FFPROBE, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(p)], capture_output=True, text=True)
	try: return float(r.stdout.strip())
	except ValueError: return float("nan")

	def main():
	ap = argparse.ArgumentParser(); ap.add_argument("--csv", default=str(ROOT / "results" / "mining" / "round2_draw_v1.csv")); ap.add_argument("--split", default="test"); ap.add_argument("--tag", default="r2"); ap.add_argument("--workers", type=int, default=6); a = ap.parse_args(); t_start = time.time()
	df = pd.read_csv(a.csv); sub = df[df.split == a.split].sort_values("draw_rank"); pre_dir = AH / f"clips_prefix_{a.tag}"; pre_dir.mkdir(exist_ok=True)
	rows, missing_full = [], []
	for r in sub.itertuples():
	idx = int(r.idx); full_cab, full_road = CLIPS_V2 / f"{idx:05d}_cab.mp4", CLIPS_V2 / f"{idx:05d}_road.mp4"
	if not (full_cab.exists() and full_road.exists()): missing_full.append(idx); continue
	p = r.uid.split("__"); leaf = "/".join(p[:3]) + "/" + "__".join(p[3:]); anchor = float(min(5.0, r.t0))
	rows.append(dict(idx=idx, uid=r.uid, t0=float(r.t0), t1=float(r.t0) + 10.0, dongle=r.dongle, dcam=str(LROOT / leaf / "dcamera.mp4"), qcam=str(LROOT / leaf / "qcamera.mp4"), clip_id=r.clip_id, split=r.split, regime=r.regime, ctrl=r.ctrl,
	anchor_s=anchor, prior_label=None, prior_label_protocol=None, pilot=False, needs_frames=bool(r.needs_frames), double_rate=True,
	selection=dict(bucket="round2_test", reason="round-2 active draw (results/mining/round2_draw_v1.csv)", draw_rank=int(r.draw_rank), stratum=r.stratum, quota_stratum=r.quota_stratum, active_score=float(r.active_score), kin_source=r.kin_source),
	clips=dict(prefix_cab=f"clips_prefix_{a.tag}/{idx:05d}_cab.mp4", prefix_road=f"clips_prefix_{a.tag}/{idx:05d}_road.mp4", full_cab=str(full_cab), full_road=str(full_road))))
	jobs = [(Path(r["clips"][f"full_{cam}"]), pre_dir / f"{r['idx']:05d}_{cam}.mp4", r["anchor_s"]) for r in rows for cam in ("cab", "road")]
	stats = {}; fails = []
	with ThreadPoolExecutor(max_workers=a.workers) as ex:
	for dst, st in ex.map(cut_prefix, jobs):
	stats[st.split(":")[0]] = stats.get(st.split(":")[0], 0) + 1
	if st.startswith("fail"): fails.append((dst.name, st))
	bad_dur = []
	for r in rows:
	for cam in ("cab", "road"):
	d = probe_dur(pre_dir / f"{r['idx']:05d}_{cam}.mp4")
	if not (d <= r["anchor_s"] + 0.1): bad_dur.append((r["idx"], cam, d))
	man = dict(generated=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), source_draw=str(Path(a.csv)), split=a.split, schema_version="h5", n=len(rows), anchor_default=5.0, double_rate_all=True,
	note="round-2 TEST manifest for the H5 server; split / prior_label / lineage never sent to the client; full clips = s15 annotate_clips_v2 (15 s, [max(0,t0-5), t0+10]); prefix physically truncated at anchor_s (libx264 crf 23); CAN evidence dirs NOT materialised here (can/<idx>.*.json to be produced by active_select.materialise_can if the server needs them)",
	clips=rows, missing_full_clips=missing_full)
	out = AH / f"clips_{a.tag}.json"; assert not out.exists(), f"{out} exists — refusing to overwrite"; json.dump(man, open(out, "w"), indent=1)
	summ = dict(split=a.split, n_clips=len(rows), missing_full_clips=missing_full, prefix_cuts=stats, cut_failures=fails, duration_violations=bad_dur, distinct_leaves=len({r["dcam"] for r in rows}), double_rate=sum(r["double_rate"] for r in rows), needs_frames=sum(r["needs_frames"] for r in rows), manifest=str(out), prefix_dir=str(pre_dir))
	write_run_json(ROOT / "results", f"cut_clips_{a.tag}", [Path(__file__)], [Path(a.csv)], None, t_start, extra={"summary": summ}, input_listing=False)
	print("CUT_DONE " + json.dumps(summ)); sys.exit(1 if (fails or bad_dur or missing_full) else 0)

	if __name__ == "__main__":
	main()