Nadeem-Shoukath115113's picture
download
raw
33.3 kB
import os
import re
import csv
import json
import random
from pathlib import Path
from typing import List, Dict, Any, Tuple, Union, Set, Optional
from collections import defaultdict
import numpy as np
import torch
def adapt_lerobot_batch_sarm(
batch: Dict[str, Any],
camera_names: List[str] = ["top_camera-images-rgb"],
eval_video: bool = False,
) -> Dict[str, Any]:
"""
Convert a batch to the (multi-stage) LeRobot-compatible format.
When eval_video=True, wrap single-example tensors with a leading batch dim
and wrap the scalar task into a single-item list to mimic batched inputs.
"""
def maybe_unsqueeze(x):
return x.unsqueeze(0) if eval_video else x
result = {
"image_frames": {},
"targets": maybe_unsqueeze(batch["targets"]),
"lengths": maybe_unsqueeze(batch["lengths"]),
"tasks": [batch["task"]] if eval_video else batch["task"],
"state": maybe_unsqueeze(batch["state"]),
"frame_relative_indices": maybe_unsqueeze(batch["frame_relative_indices"]),
}
for cam_name in camera_names:
result["image_frames"][cam_name] = maybe_unsqueeze(batch[cam_name])
return result
def adapt_lerobot_batch_rewind(
batch: dict,
camera_names: List[str] = ["top_camera-images-rgb"],
eval_video: bool = False
) -> dict:
"""Convert to lerobot-compatible batch format.
Args:
batch: Input batch dictionary.
camera_names: List of camera keys to include.
eval_video: If True, wrap tensors with an additional batch dimension.
"""
def maybe_unsqueeze(x):
return x.unsqueeze(0) if eval_video else x
result = {
"image_frames": {},
"targets": maybe_unsqueeze(batch["targets"]),
"lengths": maybe_unsqueeze(batch["lengths"]),
"tasks": batch["task"],
"state": maybe_unsqueeze(batch["state"]),
"frame_relative_indices": maybe_unsqueeze(batch["frame_relative_indices"]),
}
for cam_name in camera_names:
result["image_frames"][cam_name] = maybe_unsqueeze(batch[cam_name])
return result
def adapt_lerobot_batch_act_pri(
batch: dict,
camera_names: List[str] = ["top_camera-images-rgb"],
n_obs_steps: int = 4,
eval_video: bool = False
) -> dict:
"""Convert to lerobot-compatible batch format.
Args:
batch: Input batch dictionary.
camera_names: List of camera keys to include.
dense_annotation: Whether to transpose task annotations.
eval_video: If True, wrap tensors with an additional batch dimension.
"""
def maybe_unsqueeze(x):
return x.unsqueeze(0) if eval_video else x
result = {
"image_frames": {},
"targets": maybe_unsqueeze(batch["targets"]),
"lengths": maybe_unsqueeze(batch["lengths"]),
"steps_to_go": maybe_unsqueeze(batch["steps_to_go"]),
"tasks": batch["task"],
"state": maybe_unsqueeze(batch["state"]),
"frame_relative_indices": maybe_unsqueeze(batch["frame_relative_indices"]),
}
if "act_pri_index" in batch:
result["act_pri_index"] = maybe_unsqueeze(batch["act_pri_index"])[:, 1:n_obs_steps + 1]
# skip the first frame, it is the start of the episode
for cam_name in camera_names:
result["image_frames"][cam_name] = maybe_unsqueeze(batch[cam_name])
return result
def get_valid_episodes(repo_id: str) -> List[int]:
"""
Collects valid episode indices under the lerobot cache for the given repo_id.
Args:
repo_id (str): HuggingFace repo ID,
Returns:
List[int]: Sorted list of valid episode indices (e.g., [0, 1, 5, 7, ...])
"""
base_path = Path.home() / ".cache" / "huggingface" / "lerobot" / repo_id / "data"
episode_pattern = re.compile(r"episode_(\d+)\.parquet")
valid_episodes = []
if not base_path.exists():
raise FileNotFoundError(f"Data directory not found: {base_path}")
for chunk_dir in base_path.glob("chunk-*"):
if not chunk_dir.is_dir():
continue
for file in chunk_dir.glob("episode_*.parquet"):
match = episode_pattern.match(file.name)
if match:
ep_idx = int(match.group(1))
valid_episodes.append(ep_idx)
return sorted(valid_episodes)
def split_train_eval_episodes(valid_episodes: List[int], train_ratio: float = 0.9, seed: int = 42) -> Tuple[List[int], List[int]]:
"""
Randomly split valid episodes into training and evaluation sets.
Args:
valid_episodes (List[int]): List of valid episode indices.
train_ratio (float): Fraction of episodes to use for training (default: 0.9).
seed (int): Random seed for reproducibility (default: 42).
Returns:
Tuple[List[int], List[int]]: (train_episodes, eval_episodes)
"""
random.seed(seed)
episodes = valid_episodes.copy()
random.shuffle(episodes)
split_index = int(len(episodes) * train_ratio)
train_episodes = episodes[:split_index]
eval_episodes = episodes[split_index:]
return train_episodes, eval_episodes
# ============================================================================
# Episode-selection & evaluation analytics (ported from sarm/data_utils.py)
# ============================================================================
def find_eps_each_task(
num_per_task: int,
val_eps: List[int],
repo_id: str,
task_list: List[str],
seed: int,
) -> Dict[str, List[int]]:
"""
For each task in task_list, pick `num_per_task` episode_index.
Priority:
1) episodes whose episode_index is in `val_eps`
2) if not enough, fill with episodes for that task NOT in `val_eps` (and print warning)
Reads:
~/.cache/huggingface/lerobot/<repo_id>/meta/episodes.jsonl
Returns:
dict: {task_name: [episode_index, ...]} (length == num_per_task when possible)
"""
eps_path = (
Path("~/.cache/huggingface/lerobot")
.expanduser()
/ repo_id
/ "meta"
/ "episodes.jsonl"
)
if not eps_path.exists():
raise FileNotFoundError(f"episodes.jsonl not found: {eps_path}")
# ---- load and index episodes by task ----
by_task: Dict[str, List[int]] = {t: [] for t in task_list}
with eps_path.open("r") as f:
for line_no, line in enumerate(f, start=1):
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError as e:
raise ValueError(f"Bad JSON at line {line_no} in {eps_path}: {e}") from e
ep = obj.get("episode_index", None)
tasks = obj.get("tasks", None)
if ep is None or tasks is None:
continue
if not isinstance(tasks, list):
continue
# each episode can potentially have multiple tasks; add to any requested task
for t in tasks:
if t in by_task:
by_task[t].append(int(ep))
val_set = set(int(x) for x in val_eps)
rng = random.Random(int(seed))
out: Dict[str, List[int]] = {}
for task in task_list:
all_eps = list(dict.fromkeys(by_task.get(task, []))) # de-dup, keep order
if len(all_eps) == 0:
# task not present in this dataset (e.g. "dummy" placeholder); skip it
print(f"[WARN] task='{task}': no episodes found in {eps_path}, skipping")
continue
in_val = [e for e in all_eps if e in val_set]
not_in_val = [e for e in all_eps if e not in val_set]
picked: List[int] = []
if len(in_val) >= num_per_task:
picked = rng.sample(in_val, k=num_per_task)
else:
picked = list(in_val)
need = num_per_task - len(picked)
print(f"[WARN] task='{task}': only {len(in_val)}/{num_per_task} found")
out[task] = picked
return out
def _find_episode_dirs_with_gt(base: Path, excluded_names: Set[str], excluded_idxs: Set[int]) -> List[Path]:
"""Find episode_* directories under base that contain gt.npy, excluding specified episodes."""
eps: List[Path] = []
if not base.exists():
return eps
for p in sorted(base.iterdir()):
if not p.is_dir():
continue
if not p.name.startswith("episode_"):
continue
if not (p / "gt.npy").exists():
continue
# skip by folder name
if p.name in excluded_names:
continue
# parse idx + skip by idx
try:
idx = int(p.name.split("_")[-1])
except ValueError:
continue
if idx in excluded_idxs:
continue
eps.append(p)
return eps
def _load_pair(ep_dir: Path, pred_name: str) -> Optional[Tuple[np.ndarray, np.ndarray]]:
"""
Load {pred_name}.npy and gt.npy from an episode directory.
Returns:
(pred, gt) as float64 1D arrays with only finite entries,
truncated to same length; or None on failure.
"""
pred_path = ep_dir / f"{pred_name}.npy"
gt_path = ep_dir / "gt.npy"
if not (pred_path.exists() and gt_path.exists()):
return None
try:
pred = np.load(pred_path, allow_pickle=False)
gt = np.load(gt_path, allow_pickle=False)
pred = np.asarray(pred, dtype=np.float64).reshape(-1)
gt = np.asarray(gt, dtype=np.float64).reshape(-1)
L = min(len(pred), len(gt))
if L == 0:
return None
pred = pred[:L]
gt = gt[:L]
mask = np.isfinite(pred) & np.isfinite(gt)
if not np.any(mask):
return None
return pred[mask], gt[mask]
except Exception:
return None
def _mse(a: np.ndarray, b: np.ndarray) -> float:
diff = a - b
return float(np.mean(diff * diff))
def _format_report(title: str,
per_ep: List[Tuple[str, float, int]],
macro: float,
micro: float,
pred_name: str,
excluded_idxs: Set[int],
excluded_names: Set[str]) -> str:
lines: List[str] = []
lines.append(title)
lines.append("=" * len(title))
lines.append(f"pred file : {pred_name}.npy")
lines.append(f"gt file : gt.npy")
lines.append(f"excluded idxs : {sorted(excluded_idxs) if excluded_idxs else []}")
lines.append(f"excluded names : {sorted(excluded_names) if excluded_names else []}")
lines.append("")
if not per_ep:
lines.append("No valid episodes to aggregate.")
return "\n".join(lines) + "\n"
lines.append(f"Episodes counted : {len(per_ep)}")
lines.append(f"Macro-average MSE (mean of ep) : {macro:.6f}")
lines.append(f"Micro-average MSE (all samples) : {micro:.6f}")
lines.append("")
return "\n".join(lines) + "\n"
def compute_mse(
path,
pred_name: str = "smoothed", # or "pred"
excluded_episodes: Optional[Set[int]] = None,
report_name: str = "mse_report.txt",
verbose: bool = True,
) -> Dict:
"""
Compute MSE between `{pred_name}.npy` and `gt.npy` across episodes.
Behavior:
- If `path` is a leaf task dir (contains episode_*), scans only that dir and writes:
<path>/<report_name>
- If `path` is a root dir, scans:
- episode_* directly under root (bucket "__root__")
- episode_* under each immediate subdir (bucket = subdir name)
Writes:
<root>/<report_name> (GLOBAL)
<root>/<task>/<report_name> (per task bucket)
<root>/<report_name with __root__> (if root has episodes)
Returns a dict with global + per-bucket stats.
"""
base = Path(os.path.expanduser(str(path))).expanduser().resolve()
if not base.exists():
raise FileNotFoundError(f"Path does not exist: {base}")
excluded_idxs = set(excluded_episodes or set())
excluded_names = {f"episode_{i:06d}" for i in excluded_idxs}
# Detect if base is a single task dir: it has episodes, and it doesn't look like a root with multiple task subdirs
base_eps = _find_episode_dirs_with_gt(base, excluded_names, excluded_idxs)
has_subtask_eps = any(p.is_dir() and _find_episode_dirs_with_gt(p, excluded_names, excluded_idxs) for p in base.iterdir())
def summarize_episode_dirs(eps: List[Path]) -> Tuple[List[Tuple[str, float, int]], float, float, float, int]:
per_ep: List[Tuple[str, float, int]] = []
total_sse = 0.0
total_cnt = 0
for ep in eps:
pair = _load_pair(ep, pred_name=pred_name)
if pair is None:
continue
pred, gt = pair
n = len(pred)
if n == 0:
continue
ep_mse = _mse(pred, gt)
per_ep.append((ep.name, ep_mse, n))
diff = pred - gt
total_sse += float(np.sum(diff * diff))
total_cnt += n
if not per_ep:
return per_ep, float("nan"), float("nan"), 0.0, 0
macro = float(np.mean([m for _, m, _ in per_ep]))
micro = (total_sse / total_cnt) if total_cnt > 0 else float("nan")
return per_ep, macro, micro, total_sse, total_cnt
# -------- Case A: single task dir --------
if base_eps and not has_subtask_eps:
per_ep, macro, micro, _, _ = summarize_episode_dirs(base_eps)
report_path = base / report_name
report_path.write_text(
_format_report(
title=f"MSE Report (task='{base.name}')",
per_ep=per_ep,
macro=macro if np.isfinite(macro) else float("nan"),
micro=micro if np.isfinite(micro) else float("nan"),
pred_name=pred_name,
excluded_idxs=excluded_idxs,
excluded_names=excluded_names,
)
)
if verbose:
print(f"[OK] Wrote: {report_path}")
if per_ep:
print(f" Episodes counted: {len(per_ep)} macro={macro:.6f} micro={micro:.6f}")
else:
print(" No valid episodes.")
return {
"mode": "single_task",
"path": str(base),
"task": base.name,
"report_path": str(report_path),
"episodes_counted": len(per_ep),
"macro_mse": macro,
"micro_mse": micro,
}
# -------- Case B: root dir (multi-task buckets) --------
buckets: Dict[str, List[Path]] = {}
if base_eps:
buckets["__root__"] = base_eps
for sub in sorted(base.iterdir()):
if not sub.is_dir():
continue
eps = _find_episode_dirs_with_gt(sub, excluded_names, excluded_idxs)
if eps:
buckets[sub.name] = eps
if not buckets:
# Write empty global report
global_report_path = base / report_name
global_report_path.write_text(
_format_report(
title="MSE Report (GLOBAL)",
per_ep=[],
macro=float("nan"),
micro=float("nan"),
pred_name=pred_name,
excluded_idxs=excluded_idxs,
excluded_names=excluded_names,
)
)
if verbose:
print(f"[WARN] No episodes with gt.npy found under: {base}")
print(f"[OK] Wrote empty global report: {global_report_path}")
return {"mode": "root", "path": str(base), "buckets": {}, "global_report_path": str(global_report_path)}
# Per-bucket stats + global merge (micro via total_sse/total_cnt; macro via mean of per-episode MSEs)
bucket_stats = {}
all_per_ep: List[Tuple[str, float, int]] = []
global_total_sse = 0.0
global_total_cnt = 0
for name, eps in buckets.items():
per_ep, macro, micro, sse, cnt = summarize_episode_dirs(eps)
# write per-bucket report
if name == "__root__":
report_path = base / report_name.replace(".txt", "__root__.txt")
title = "MSE Report (bucket='__root__')"
else:
report_path = (base / name) / report_name
title = f"MSE Report (task='{name}')"
report_path.write_text(
_format_report(
title=title,
per_ep=per_ep,
macro=macro if np.isfinite(macro) else float("nan"),
micro=micro if np.isfinite(micro) else float("nan"),
pred_name=pred_name,
excluded_idxs=excluded_idxs,
excluded_names=excluded_names,
)
)
if verbose:
print(f"[OK] Wrote: {report_path}")
bucket_stats[name] = {
"report_path": str(report_path),
"episodes_counted": len(per_ep),
"macro_mse": macro,
"micro_mse": micro,
}
# merge into global
all_per_ep.extend([(f"{name}/{ep}", m, n) for ep, m, n in per_ep])
global_total_sse += sse
global_total_cnt += cnt
if all_per_ep:
global_macro = float(np.mean([m for _, m, _ in all_per_ep]))
global_micro = (global_total_sse / global_total_cnt) if global_total_cnt > 0 else float("nan")
else:
global_macro = float("nan")
global_micro = float("nan")
global_report_path = base / report_name
global_report_path.write_text(
_format_report(
title="MSE Report (GLOBAL)",
per_ep=[(ep, m, n) for ep, m, n in all_per_ep],
macro=global_macro,
micro=global_micro,
pred_name=pred_name,
excluded_idxs=excluded_idxs,
excluded_names=excluded_names,
)
)
if verbose:
print(f"[OK] Wrote: {global_report_path}")
if all_per_ep:
print(f" GLOBAL episodes counted: {len(all_per_ep)} macro={global_macro:.6f} micro={global_micro:.6f}")
else:
print(" GLOBAL: no valid episodes.")
return {
"mode": "root",
"path": str(base),
"global_report_path": str(global_report_path),
"global_episodes_counted": len(all_per_ep),
"global_macro_mse": global_macro,
"global_micro_mse": global_micro,
"buckets": bucket_stats,
}
def _load_act_pri_pair(ep_dir: Path) -> Optional[Tuple[np.ndarray, np.ndarray]]:
"""Load pred_act_pri.npy and gt_act_pri.npy from an episode dir."""
pred_path = ep_dir / "pred_act_pri.npy"
gt_path = ep_dir / "gt_act_pri.npy"
if not (pred_path.exists() and gt_path.exists()):
return None
try:
pred = np.asarray(np.load(pred_path, allow_pickle=False), dtype=np.int64).reshape(-1)
gt = np.asarray(np.load(gt_path, allow_pickle=False), dtype=np.int64).reshape(-1)
L = min(len(pred), len(gt))
if L == 0:
return None
return pred[:L], gt[:L]
except Exception:
return None
def _find_episode_dirs_with_act_pri(base: Path) -> List[Path]:
eps: List[Path] = []
if not base.exists():
return eps
for p in sorted(base.iterdir()):
if not p.is_dir() or not p.name.startswith("episode_"):
continue
if not (p / "pred_act_pri.npy").exists() or not (p / "gt_act_pri.npy").exists():
continue
eps.append(p)
return eps
def _format_act_pri_report(title: str,
per_ep: List[Tuple[str, Dict[str, Any]]],
agg: Dict[str, float],
dummy_task_idx: int) -> str:
lines: List[str] = []
lines.append(title)
lines.append("=" * len(title))
lines.append(f"dummy task index : {dummy_task_idx}")
lines.append("")
if not per_ep:
lines.append("No valid episodes to aggregate.")
return "\n".join(lines) + "\n"
lines.append(f"Episodes counted : {len(per_ep)}")
lines.append(f"ActPri accuracy (vanilla) : {agg['task_acc_vanilla']:.6f} N={agg['n_vanilla']}")
lines.append(f"ActPri accuracy (no-dummy GT) : {agg['task_acc_nodummy']:.6f} N={agg['n_nodummy']}")
lines.append(f"Class accuracy (vanilla) : {agg['class_acc_vanilla']:.6f} N={agg['n_vanilla']}")
lines.append(f"Class accuracy (no-dummy GT) : {agg['class_acc_nodummy']:.6f} N={agg['n_nodummy']}")
lines.append("")
return "\n".join(lines) + "\n"
def compute_act_pri_accuracy(
path,
task_to_class_id,
dummy_task_idx: int,
report_name: str = "act_pri_accuracy_report.txt",
verbose: bool = True,
) -> Dict:
"""
Compute act_pri (task) and class estimation accuracy across episodes.
Reads pred_act_pri.npy / gt_act_pri.npy under episode_* dirs.
Reports both vanilla accuracy and the variant that excludes samples
whose GT task index equals `dummy_task_idx`.
Behavior mirrors compute_mse:
- If `path` is a leaf task dir, writes <path>/<report_name>.
- If `path` is a root dir, writes per-task and a GLOBAL report.
"""
base = Path(os.path.expanduser(str(path))).expanduser().resolve()
if not base.exists():
raise FileNotFoundError(f"Path does not exist: {base}")
if isinstance(task_to_class_id, torch.Tensor):
t2c = task_to_class_id.detach().cpu().numpy().astype(np.int64)
else:
t2c = np.asarray(task_to_class_id, dtype=np.int64)
def _per_ep_stats(pred: np.ndarray, gt: np.ndarray) -> Dict[str, Any]:
pred_c = t2c[pred]
gt_c = t2c[gt]
task_ok = (pred == gt)
class_ok = (pred_c == gt_c)
mask = (gt != dummy_task_idx)
n = int(len(gt))
n_nd = int(mask.sum())
return {
"n": n,
"n_nd": n_nd,
"task_correct": int(task_ok.sum()),
"class_correct": int(class_ok.sum()),
"task_correct_nd": int((task_ok & mask).sum()),
"class_correct_nd": int((class_ok & mask).sum()),
"task_acc": float(task_ok.mean()) if n > 0 else float("nan"),
"class_acc": float(class_ok.mean()) if n > 0 else float("nan"),
"task_acc_nd": float((task_ok & mask).sum() / n_nd) if n_nd > 0 else float("nan"),
"class_acc_nd": float((class_ok & mask).sum() / n_nd) if n_nd > 0 else float("nan"),
}
def _aggregate(per_ep: List[Tuple[str, Dict[str, Any]]]) -> Dict[str, float]:
if not per_ep:
nan = float("nan")
return {
"task_acc_vanilla": nan, "class_acc_vanilla": nan,
"task_acc_nodummy": nan, "class_acc_nodummy": nan,
"n_vanilla": 0, "n_nodummy": 0,
}
# micro accuracy: sum correct / sum count across episodes
total_n = sum(s["n"] for _, s in per_ep)
total_n_nd = sum(s["n_nd"] for _, s in per_ep)
task_acc_v = (sum(s["task_correct"] for _, s in per_ep) / total_n) if total_n > 0 else float("nan")
class_acc_v = (sum(s["class_correct"] for _, s in per_ep) / total_n) if total_n > 0 else float("nan")
task_acc_nd = (sum(s["task_correct_nd"] for _, s in per_ep) / total_n_nd) if total_n_nd > 0 else float("nan")
class_acc_nd = (sum(s["class_correct_nd"] for _, s in per_ep) / total_n_nd) if total_n_nd > 0 else float("nan")
return {
"task_acc_vanilla": task_acc_v, "class_acc_vanilla": class_acc_v,
"task_acc_nodummy": task_acc_nd, "class_acc_nodummy": class_acc_nd,
"n_vanilla": total_n, "n_nodummy": total_n_nd,
}
def summarize_episode_dirs(eps: List[Path]) -> Tuple[List[Tuple[str, Dict[str, Any]]], Dict[str, float]]:
per_ep: List[Tuple[str, Dict[str, Any]]] = []
for ep in eps:
pair = _load_act_pri_pair(ep)
if pair is None:
continue
pred, gt = pair
per_ep.append((ep.name, _per_ep_stats(pred, gt)))
return per_ep, _aggregate(per_ep)
base_eps = _find_episode_dirs_with_act_pri(base)
has_subtask_eps = any(
p.is_dir() and _find_episode_dirs_with_act_pri(p) for p in base.iterdir()
)
# -------- Case A: single task dir --------
if base_eps and not has_subtask_eps:
per_ep, agg = summarize_episode_dirs(base_eps)
report_path = base / report_name
report_path.write_text(
_format_act_pri_report(
title=f"ActPri Accuracy Report (task='{base.name}')",
per_ep=per_ep,
agg=agg,
dummy_task_idx=dummy_task_idx,
)
)
if verbose:
print(f"[OK] Wrote: {report_path}")
return {
"mode": "single_task",
"path": str(base),
"task": base.name,
"report_path": str(report_path),
"episodes_counted": len(per_ep),
**agg,
}
# -------- Case B: root dir (multi-task buckets) --------
buckets: Dict[str, List[Path]] = {}
if base_eps:
buckets["__root__"] = base_eps
for sub in sorted(base.iterdir()):
if not sub.is_dir():
continue
eps = _find_episode_dirs_with_act_pri(sub)
if eps:
buckets[sub.name] = eps
if not buckets:
global_report_path = base / report_name
global_report_path.write_text(
_format_act_pri_report(
title="ActPri Accuracy Report (GLOBAL)",
per_ep=[],
agg=_aggregate([]),
dummy_task_idx=dummy_task_idx,
)
)
if verbose:
print(f"[WARN] No episodes with pred/gt_act_pri.npy under: {base}")
return {"mode": "root", "path": str(base), "buckets": {}, "global_report_path": str(global_report_path)}
bucket_stats: Dict[str, Any] = {}
all_per_ep: List[Tuple[str, Dict[str, Any]]] = []
for name, eps in buckets.items():
per_ep, agg = summarize_episode_dirs(eps)
if name == "__root__":
report_path = base / report_name.replace(".txt", "__root__.txt")
title = "ActPri Accuracy Report (bucket='__root__')"
else:
report_path = (base / name) / report_name
title = f"ActPri Accuracy Report (task='{name}')"
report_path.write_text(
_format_act_pri_report(
title=title,
per_ep=per_ep,
agg=agg,
dummy_task_idx=dummy_task_idx,
)
)
if verbose:
print(f"[OK] Wrote: {report_path}")
bucket_stats[name] = {"report_path": str(report_path), "episodes_counted": len(per_ep), **agg}
all_per_ep.extend([(f"{name}/{ep}", s) for ep, s in per_ep])
global_agg = _aggregate(all_per_ep)
global_report_path = base / report_name
global_report_path.write_text(
_format_act_pri_report(
title="ActPri Accuracy Report (GLOBAL)",
per_ep=all_per_ep,
agg=global_agg,
dummy_task_idx=dummy_task_idx,
)
)
if verbose:
print(f"[OK] Wrote: {global_report_path}")
return {
"mode": "root",
"path": str(base),
"global_report_path": str(global_report_path),
"global_episodes_counted": len(all_per_ep),
"buckets": bucket_stats,
**{f"global_{k}": v for k, v in global_agg.items()},
}
_RE_ACT_PRI_V = re.compile(r"^ActPri\s+accuracy \(vanilla\)\s*:\s*([0-9.eE+-]+)")
_RE_ACT_PRI_ND = re.compile(r"^ActPri\s+accuracy \(no-dummy GT\)\s*:\s*([0-9.eE+-]+)")
_RE_CLASS_V = re.compile(r"^Class\s+accuracy \(vanilla\)\s*:\s*([0-9.eE+-]+)")
_RE_CLASS_ND = re.compile(r"^Class\s+accuracy \(no-dummy GT\)\s*:\s*([0-9.eE+-]+)")
def _parse_act_pri_report(txt_path: Path) -> Optional[Dict[str, float]]:
out: Dict[str, float] = {}
patterns = {
"act_pri_vanilla": _RE_ACT_PRI_V,
"act_pri_nodummy": _RE_ACT_PRI_ND,
"class_vanilla": _RE_CLASS_V,
"class_nodummy": _RE_CLASS_ND,
}
for line in txt_path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
for key, regex in patterns.items():
if key in out:
continue
m = regex.match(line)
if m:
out[key] = float(m.group(1))
break
return out if out else None
def collect_task_act_pri_accuracy_to_csv(
eval_folder: str | Path,
task_sequence_list: Optional[List[str]] = None,
) -> Path:
"""
Walks `eval_folder/<task>/act_pri_accuracy_report.txt`, collects the 4 accuracy
numbers per task, and writes a CSV with 5 rows:
row 0: task names
row 1: ActPri accuracy (vanilla)
row 2: ActPri accuracy (no-dummy GT)
row 3: Class accuracy (vanilla)
row 4: Class accuracy (no-dummy GT)
"""
eval_folder = Path(eval_folder).expanduser().resolve()
out_csv = eval_folder / "act_pri_accuracy_summary.csv"
temp_data: Dict[str, Dict[str, float]] = {}
for task_dir in eval_folder.iterdir():
if not task_dir.is_dir():
continue
report_path = task_dir / "act_pri_accuracy_report.txt"
if not report_path.exists():
continue
parsed = _parse_act_pri_report(report_path)
if parsed:
temp_data[task_dir.name] = parsed
if task_sequence_list is not None:
ordered_tasks = [t for t in task_sequence_list if t in temp_data]
else:
ordered_tasks = sorted(temp_data.keys())
nan = float("nan")
tasks: List[str] = []
act_pri_v: List[float] = []
act_pri_nd: List[float] = []
class_v: List[float] = []
class_nd: List[float] = []
for t in ordered_tasks:
tasks.append(t)
d = temp_data[t]
act_pri_v.append(d.get("act_pri_vanilla", nan))
act_pri_nd.append(d.get("act_pri_nodummy", nan))
class_v.append(d.get("class_vanilla", nan))
class_nd.append(d.get("class_nodummy", nan))
with out_csv.open("w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(tasks)
w.writerow(act_pri_v)
w.writerow(act_pri_nd)
w.writerow(class_v)
w.writerow(class_nd)
return out_csv
_RE_EPISODES = re.compile(r"^Episodes counted\s*:\s*(\d+)\s*$")
_RE_MACRO = re.compile(r"^Macro-average MSE .*:\s*([0-9.eE+-]+)\s*$")
_RE_MICRO = re.compile(r"^Micro-average MSE .*:\s*([0-9.eE+-]+)\s*$")
def _parse_mse_report(txt_path: Path) -> Optional[Dict[str, Any]]:
episodes = None
macro = None
micro = None
for line in txt_path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
m = _RE_EPISODES.match(line)
if m:
episodes = int(m.group(1))
continue
m = _RE_MACRO.match(line)
if m:
macro = float(m.group(1))
continue
m = _RE_MICRO.match(line)
if m:
micro = float(m.group(1))
continue
if macro is None and micro is None and episodes is None:
return None
return {
"episodes": episodes,
"macro_mse": macro,
"micro_mse": micro,
"report_path": str(txt_path),
}
def collect_task_mse_to_csv(eval_folder: str | Path,
task_sequence_list: Optional[List[str]]) -> Path:
"""
Output CSV with exactly 3 rows.
If task_sequence_list is provided, rows will follow that specific task order.
"""
eval_folder = Path(eval_folder).expanduser().resolve()
out_csv = eval_folder / "mse_summary.csv"
# Use a dictionary to store parsed data: { task_name: {"macro": float, "micro": float} }
temp_data: Dict[str, Dict[str, float]] = {}
# 1. Collect all available data from the folder
for task_dir in eval_folder.iterdir():
if not task_dir.is_dir():
continue
report_path = task_dir / "mse_report.txt"
if not report_path.exists():
continue
parsed = _parse_mse_report(report_path)
if parsed:
temp_data[task_dir.name] = {
"macro_mse": parsed["macro_mse"],
"micro_mse": parsed["micro_mse"]
}
# 2. Determine the final order
if task_sequence_list is not None:
# Use the provided sequence, but only include tasks that actually exist in temp_data
ordered_tasks = [t for t in task_sequence_list if t in temp_data]
else:
# Default: alphanumeric sort of the found tasks
ordered_tasks = sorted(temp_data.keys())
# 3. Prepare rows for CSV
tasks: List[str] = []
macro_mse: List[float] = []
micro_mse: List[float] = []
for t_name in ordered_tasks:
tasks.append(t_name)
macro_mse.append(temp_data[t_name]["macro_mse"])
micro_mse.append(temp_data[t_name]["micro_mse"])
# 4. Write to CSV
with out_csv.open("w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(tasks)
w.writerow(macro_mse)
w.writerow(micro_mse)
return out_csv

Xet Storage Details

Size:
33.3 kB
·
Xet hash:
02ed2d3f73ac288633efa97e8e5456f9f76f336ed1497e32b13b10ac25b6caa6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.