Buckets:
| """Evaluate camera-motion adherence of generated videos against a camera reference clip. | |
| This is CamCloneMaster's camera-accuracy protocol (arXiv 2506.03140, Sec. 5.1): estimate | |
| per-frame camera poses from both videos with MegaSaM, then score the two trajectories with | |
| CamI2V's three errors. | |
| RotErr sum of geodesic rotation error, radians (lower is better) | |
| TransErr sum of scale-normalized translation error (lower is better) | |
| CamMC sum of combined [R|t] error (lower is better) | |
| The scene content of a generated clip and its camera reference is different, and that is | |
| fine - camera motion is scene-independent, and both trajectories are expressed relative to | |
| their own first frame before being compared. | |
| Each metric is also reported against the **static-camera control**: the do-nothing shot, | |
| identity pose at every frame, scored against the same reference. That control is the chance | |
| level for this condition and is emitted as ``<metric>_chance``, so no ``--shuffle-refs`` pass | |
| is needed - it is exact, analytic and free. | |
| Reading the raw errors without the control is a mistake this metric invites. References | |
| differ enormously in motion magnitude, so a raw RotErr partly reports which camera the | |
| sample drew. ``RotErr_rel = RotErr / RotErr_chance`` puts them on one scale: < 1 means the | |
| generated shot beat a locked-off camera, >= 1 means it did not. | |
| Known limitations: | |
| * MegaSaM is a monocular reconstruction. On a low-parallax or heavily dynamic clip it can | |
| return a near-degenerate trajectory; those samples are flagged via ``degenerate`` and | |
| ``ref_max_trans`` / ``gen_max_trans`` rather than reported as if they were fine. | |
| * Absolute scale is unrecoverable from monocular video, so translations are normalized per | |
| trajectory. TransErr therefore measures the SHAPE of the translation path, not its size. | |
| * Camera motion is all this sees - not scene coherence, temporal flicker, or quality. | |
| Usage: | |
| python eval_camera_new/eval.py \\ | |
| --generated-dir /path/to/generated/videos \\ | |
| --dataset-dir /path/to/dataset/camera \\ | |
| --output-dir /path/to/output | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import shutil | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from megasam import MegaSamError, estimate_poses # noqa: E402 | |
| from metric import camera_metrics, eval_metrics, static_control # noqa: E402 | |
| # Applied to the control's PER-FRAME value, so it is a statement about the reference and | |
| # not about clip length: below this the reference barely moves in that degree of freedom, | |
| # the locked-off camera is already almost a perfect match, and <metric>_rel becomes a ratio | |
| # to a meaningless denominator. 1e-3 rad/frame is 0.06 deg/frame of rotation, and the same | |
| # figure in the scale-normalized translation units. | |
| # | |
| # This matters in practice rather than in principle: one sample in the v1 camera set has a | |
| # control RotErr of 0.026 rad over 41 frames, and dividing by it produced a RotErr_rel of | |
| # 71 that pulled the mean over 20 samples from 0.62 to 4.46. Such samples now report None, | |
| # which the mean skips, and are counted in <metric>_rel_undefined. | |
| _CONTROL_FLOOR_PER_FRAME = 1e-3 | |
| _REPORT_KEYS = ( | |
| eval_metrics | |
| + [f"{m}_per_frame" for m in eval_metrics] | |
| + [f"{m}_chance" for m in eval_metrics] | |
| + [f"{m}_rel" for m in eval_metrics] | |
| ) | |
| def evaluate_camera( | |
| generated_dir: Path, | |
| dataset_dir: Path, | |
| output_dir: Path | None = None, | |
| pose_cache_dir: Path | None = None, | |
| work_dir: Path | None = None, | |
| megasam_root: Path | None = None, | |
| num_frames: int | None = None, | |
| buffer: int = 1024, | |
| overwrite: bool = False, | |
| keep_work: bool = False, | |
| shuffle_refs: bool = False, | |
| ) -> dict: | |
| generated_files = {f.stem: f for f in sorted(generated_dir.glob("*.mp4"))} | |
| reference_files = {f.stem: f for f in sorted(dataset_dir.glob("*.mp4"))} | |
| sample_ids = sorted(set(generated_files) & set(reference_files)) | |
| if not sample_ids: | |
| print(f"No matching samples between {generated_dir} and {dataset_dir}") | |
| return {} | |
| if shuffle_refs: | |
| # Accepted for interface uniformity with the other evaluators. The camera chance | |
| # level is the analytic static-camera control, which is exact and costs nothing; | |
| # a shuffled reference would be a noisier estimate of the same idea. | |
| print("[eval_camera] --shuffle-refs: chance is the static-camera control; no shuffle needed.") | |
| base = output_dir or Path.cwd() | |
| pose_cache_dir = pose_cache_dir or base / "poses" | |
| work_dir = work_dir or base / "_work" | |
| work_dir.mkdir(parents=True, exist_ok=True) | |
| def poses(video: Path) -> np.ndarray: | |
| return estimate_poses( | |
| video, pose_cache_dir, work_dir, megasam_root, | |
| buffer=buffer, overwrite=overwrite, keep_work=keep_work, | |
| )["cam_c2w"] | |
| print(f"Evaluating {len(sample_ids)} sample(s)...") | |
| per_sample: dict[str, dict] = {} | |
| failed: dict[str, str] = {} | |
| for i, sid in enumerate(sample_ids, 1): | |
| started = time.time() | |
| try: | |
| ref_c2w = poses(reference_files[sid]) | |
| gen_c2w = poses(generated_files[sid]) | |
| except (MegaSamError, ValueError) as exc: | |
| failed[sid] = str(exc).splitlines()[0] | |
| print(f" [{i}/{len(sample_ids)}] {sid}: SKIPPED ({failed[sid]})") | |
| continue | |
| row = camera_metrics(ref_c2w, gen_c2w, num_frames=num_frames) | |
| control = static_control(ref_c2w, num_frames=row["n_frames"]) | |
| for m in eval_metrics: | |
| row[f"{m}_chance"] = control[m] | |
| informative = control[m] / row["n_frames"] > _CONTROL_FLOOR_PER_FRAME | |
| row[f"{m}_rel"] = row[m] / control[m] if informative else None | |
| per_sample[sid] = row | |
| print( | |
| f" [{i}/{len(sample_ids)}] {sid}: " | |
| + " ".join(f"{m}={row[m]:.3f} (ctl {row[f'{m}_chance']:.3f})" for m in eval_metrics) | |
| + f" [{row['n_frames']}f, {time.time() - started:.0f}s]" | |
| + (" DEGENERATE" if row["degenerate"] else "") | |
| ) | |
| if not keep_work: | |
| shutil.rmtree(work_dir, ignore_errors=True) | |
| if not per_sample: | |
| print("No samples scored.") | |
| return {} | |
| scored_ids = sorted(per_sample) | |
| mean, median, undefined = {}, {}, {} | |
| for key in _REPORT_KEYS: | |
| vals = [per_sample[s][key] for s in scored_ids if per_sample[s].get(key) is not None] | |
| mean[key] = float(np.mean(vals)) if vals else float("nan") | |
| # A ratio distribution is heavy-tailed by construction - one reference with mild | |
| # motion dominates a mean of ratios - so the median is the honest headline for the | |
| # _rel columns and is reported next to it rather than instead of it. | |
| median[key] = float(np.median(vals)) if vals else float("nan") | |
| undefined[key] = len(scored_ids) - len(vals) | |
| beats = {m: sum(1 for s in scored_ids if per_sample[s][m] < per_sample[s][f"{m}_chance"]) for m in eval_metrics} | |
| degenerate = [s for s in scored_ids if per_sample[s]["degenerate"]] | |
| print(f"\n{'=' * 60}") | |
| print(f"Mean metrics over {len(scored_ids)} sample(s):") | |
| for key in _REPORT_KEYS: | |
| extra = f" (median {median[key]:.6f})" if key.endswith("_rel") else "" | |
| skipped = f" [{undefined[key]} undefined]" if undefined[key] else "" | |
| print(f" {key}: {mean[key]:.6f}{extra}{skipped}") | |
| for m in eval_metrics: | |
| print(f" beats static control ({m}): {beats[m]}/{len(scored_ids)}") | |
| if degenerate: | |
| print(f" DEGENERATE reconstructions ({len(degenerate)}): {', '.join(degenerate)}") | |
| if failed: | |
| print(f" failed ({len(failed)}): {', '.join(sorted(failed))}") | |
| result = { | |
| "mean": {k: (None if v != v else round(v, 6)) for k, v in mean.items()}, | |
| "median": {k: (None if v != v else round(v, 6)) for k, v in median.items()}, | |
| "rel_undefined": {f"{m}_rel": undefined[f"{m}_rel"] for m in eval_metrics}, | |
| "beats_control": {m: f"{beats[m]}/{len(scored_ids)}" for m in eval_metrics}, | |
| "degenerate": degenerate, | |
| "failed": failed, | |
| "per_sample": { | |
| sid: {k: (None if v is None else round(v, 6) if isinstance(v, float) else v) | |
| for k, v in per_sample[sid].items()} | |
| for sid in scored_ids | |
| }, | |
| } | |
| if output_dir is None: | |
| return result | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| (output_dir / "results.json").write_text(json.dumps(result, indent=2)) | |
| with (output_dir / "results.csv").open("w", newline="") as handle: | |
| writer = csv.writer(handle) | |
| writer.writerow(["sample_id"] + _REPORT_KEYS) | |
| writer.writerow(["average"] + [f"{mean[k]:.6f}" for k in _REPORT_KEYS]) | |
| for sid in scored_ids: | |
| writer.writerow( | |
| [sid] + ["" if per_sample[sid].get(k) is None else f"{per_sample[sid][k]:.6f}" | |
| for k in _REPORT_KEYS] | |
| ) | |
| print(f"Results saved to {output_dir}") | |
| return result | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description="Evaluate camera-trajectory adherence (MegaSaM poses + RotErr/TransErr/CamMC).", | |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter, | |
| ) | |
| parser.add_argument("--generated-dir", required=True, | |
| help="Directory containing generated .mp4 files (one per sample ID).") | |
| parser.add_argument("--dataset-dir", required=True, | |
| help="Directory of camera reference videos, i.e. <dataset>/camera. Expects " | |
| "<id>.mp4 whose stems match the generated video stems.") | |
| parser.add_argument("--output-dir", default=None, | |
| help="Folder to write results.json and results.csv.") | |
| parser.add_argument("--pose-cache-dir", default=None, | |
| help="Where MegaSaM trajectories are cached, keyed by video content hash. " | |
| "Default: <output-dir>/poses. Reuse across runs to skip re-estimation.") | |
| parser.add_argument("--work-dir", default=None, | |
| help="Scratch for frames and depth priors. Default: <output-dir>/_work, deleted at the end.") | |
| parser.add_argument("--megasam-root", default=None, help="Default: <repo>/tools/mega-sam") | |
| parser.add_argument("--num-frames", type=int, default=None, | |
| help="Resample both trajectories to this many frames. Default: the shorter of the two.") | |
| parser.add_argument("--buffer", type=int, default=1024, help="DROID-SLAM keyframe buffer size.") | |
| parser.add_argument("--overwrite", action="store_true", help="Re-estimate poses even on a cache hit.") | |
| parser.add_argument("--keep-work", action="store_true", help="Keep frames/depth intermediates for debugging.") | |
| parser.add_argument("--shuffle-refs", action="store_true", | |
| help="Accepted for uniformity with the other evaluators. The camera chance level is the " | |
| "analytic static-camera control, always emitted as <metric>_chance, so this flag " | |
| "changes nothing here.") | |
| args = parser.parse_args() | |
| generated_dir = Path(args.generated_dir) | |
| dataset_dir = Path(args.dataset_dir) | |
| if not generated_dir.exists(): | |
| parser.error(f"Generated video directory not found: {generated_dir}") | |
| if not dataset_dir.exists(): | |
| parser.error(f"Camera reference directory not found: {dataset_dir}") | |
| evaluate_camera( | |
| generated_dir=generated_dir, | |
| dataset_dir=dataset_dir, | |
| output_dir=Path(args.output_dir) if args.output_dir else None, | |
| pose_cache_dir=Path(args.pose_cache_dir) if args.pose_cache_dir else None, | |
| work_dir=Path(args.work_dir) if args.work_dir else None, | |
| megasam_root=Path(args.megasam_root) if args.megasam_root else None, | |
| num_frames=args.num_frames, | |
| buffer=args.buffer, | |
| overwrite=args.overwrite, | |
| keep_work=args.keep_work, | |
| shuffle_refs=args.shuffle_refs, | |
| ) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 12.2 kB
- Xet hash:
- 5c749b6fae20941efd8e58cf1a794fa3e914bac351680d7c0b4757ade2875a3c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.