Add files using upload-large-folder tool
Browse files- backfill_stage_cache_costs.py +80 -0
- drop_net_outputs.py +83 -0
- extract_gvhmr_tars.sh +144 -0
- fast_rm.sh +279 -0
- oss_download.sh +67 -0
- oss_upload.sh +33 -0
- read_pt.py +42 -0
- repair_cache_jsonl.py +143 -0
- repair_math_cache_answers.py +269 -0
- report_stage_costs.py +622 -0
- report_stage_recovery.py +327 -0
- tensorrt.sh +14 -0
backfill_stage_cache_costs.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Backfill or recompute cost fields for all staged benchmark cache jsonl files."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Dict, Iterable, List
|
| 11 |
+
|
| 12 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 13 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 14 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 15 |
+
|
| 16 |
+
from answer_utils import repair_confidence_reason_record_fields
|
| 17 |
+
from cascade_utils import attach_dynamic_costs
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _read_jsonl(path: Path) -> List[Dict]:
|
| 21 |
+
rows: List[Dict] = []
|
| 22 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 23 |
+
for line in handle:
|
| 24 |
+
line = line.strip()
|
| 25 |
+
if line:
|
| 26 |
+
rows.append(repair_confidence_reason_record_fields(json.loads(line)))
|
| 27 |
+
return rows
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _write_jsonl(path: Path, rows: List[Dict]) -> None:
|
| 31 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 32 |
+
with path.open("w", encoding="utf-8") as handle:
|
| 33 |
+
for row in rows:
|
| 34 |
+
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _iter_cache_files(root: Path) -> Iterable[Path]:
|
| 38 |
+
for path in sorted(root.rglob("*.stage*.cached.jsonl")):
|
| 39 |
+
if path.is_file():
|
| 40 |
+
yield path
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def main() -> int:
|
| 44 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 45 |
+
parser.add_argument("--root", default="cache/benchmarks", help="Stage cache root directory.")
|
| 46 |
+
parser.add_argument(
|
| 47 |
+
"--missing-only",
|
| 48 |
+
action="store_true",
|
| 49 |
+
help="Only fill missing cost fields. Default behavior is to recompute all cost fields with current pricing.",
|
| 50 |
+
)
|
| 51 |
+
args = parser.parse_args()
|
| 52 |
+
|
| 53 |
+
root = Path(args.root)
|
| 54 |
+
if not root.exists():
|
| 55 |
+
raise FileNotFoundError(f"Cache root does not exist: {root}")
|
| 56 |
+
|
| 57 |
+
force_recompute = not args.missing_only
|
| 58 |
+
total_files = 0
|
| 59 |
+
total_rows = 0
|
| 60 |
+
|
| 61 |
+
for path in _iter_cache_files(root):
|
| 62 |
+
rows = _read_jsonl(path)
|
| 63 |
+
if not rows:
|
| 64 |
+
print(f"[Skip] empty {path}")
|
| 65 |
+
continue
|
| 66 |
+
changed = attach_dynamic_costs(rows, force_recompute=force_recompute)
|
| 67 |
+
if changed:
|
| 68 |
+
_write_jsonl(path, rows)
|
| 69 |
+
print(f"[Done] {path} rows={len(rows)} recompute={'yes' if force_recompute else 'missing-only'}")
|
| 70 |
+
else:
|
| 71 |
+
print(f"[Skip] up-to-date {path} rows={len(rows)}")
|
| 72 |
+
total_files += 1
|
| 73 |
+
total_rows += len(rows)
|
| 74 |
+
|
| 75 |
+
print(f"[Summary] files={total_files} rows={total_rows} mode={'recompute' if force_recompute else 'missing-only'}")
|
| 76 |
+
return 0
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
if __name__ == "__main__":
|
| 80 |
+
raise SystemExit(main())
|
drop_net_outputs.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import argparse
|
| 5 |
+
import os
|
| 6 |
+
import torch
|
| 7 |
+
from tqdm import tqdm
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def load_pt(path: Path):
|
| 11 |
+
try:
|
| 12 |
+
return torch.load(path, map_location="cpu", weights_only=False)
|
| 13 |
+
except TypeError:
|
| 14 |
+
return torch.load(path, map_location="cpu")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def iter_pt_files(root: Path):
|
| 18 |
+
for p in root.rglob("*.pt"):
|
| 19 |
+
if p.is_file():
|
| 20 |
+
yield p
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def process_one(path: Path, key: str):
|
| 24 |
+
try:
|
| 25 |
+
obj = load_pt(path)
|
| 26 |
+
if isinstance(obj, dict) and key in obj:
|
| 27 |
+
obj.pop(key, None)
|
| 28 |
+
torch.save(obj, path)
|
| 29 |
+
return 1, 0
|
| 30 |
+
return 0, 0
|
| 31 |
+
except Exception:
|
| 32 |
+
return 0, 1
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def main():
|
| 36 |
+
parser = argparse.ArgumentParser(description="Drop a key from all .pt files recursively.")
|
| 37 |
+
parser.add_argument("root", nargs="?", default="/cpfs01/videogen/gvhmr_smpl")
|
| 38 |
+
parser.add_argument("--key", default="net_outputs")
|
| 39 |
+
parser.add_argument("--workers", type=int, default=min(16, os.cpu_count() or 1))
|
| 40 |
+
parser.add_argument("--prefetch", type=int, default=4)
|
| 41 |
+
args = parser.parse_args()
|
| 42 |
+
|
| 43 |
+
root = Path(args.root)
|
| 44 |
+
files = iter_pt_files(root)
|
| 45 |
+
workers = max(1, args.workers)
|
| 46 |
+
inflight_cap = max(1, workers * max(1, args.prefetch))
|
| 47 |
+
scanned = 0
|
| 48 |
+
removed = 0
|
| 49 |
+
errors = 0
|
| 50 |
+
|
| 51 |
+
print(f"start: root={root}, workers={workers}", flush=True)
|
| 52 |
+
with ThreadPoolExecutor(max_workers=workers) as pool, tqdm(
|
| 53 |
+
desc="processing", unit="file", dynamic_ncols=True
|
| 54 |
+
) as pbar:
|
| 55 |
+
inflight = set()
|
| 56 |
+
for _ in range(inflight_cap):
|
| 57 |
+
try:
|
| 58 |
+
f = next(files)
|
| 59 |
+
except StopIteration:
|
| 60 |
+
break
|
| 61 |
+
inflight.add(pool.submit(process_one, f, args.key))
|
| 62 |
+
|
| 63 |
+
while inflight:
|
| 64 |
+
done, inflight = wait(inflight, return_when=FIRST_COMPLETED)
|
| 65 |
+
for fut in done:
|
| 66 |
+
r, e = fut.result()
|
| 67 |
+
scanned += 1
|
| 68 |
+
removed += r
|
| 69 |
+
errors += e
|
| 70 |
+
pbar.update(len(done))
|
| 71 |
+
|
| 72 |
+
for _ in range(len(done)):
|
| 73 |
+
try:
|
| 74 |
+
f = next(files)
|
| 75 |
+
except StopIteration:
|
| 76 |
+
break
|
| 77 |
+
inflight.add(pool.submit(process_one, f, args.key))
|
| 78 |
+
|
| 79 |
+
print(f"done: {scanned} files scanned, {removed} files updated, {errors} errors")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
if __name__ == "__main__":
|
| 83 |
+
main()
|
extract_gvhmr_tars.sh
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
SRC_DIR="${SRC_DIR:-/cpfs01/videogen/gvhmr_res}"
|
| 5 |
+
DST_DIR="${DST_DIR:-/cpfs01/videogen/gvhmr_smpl}"
|
| 6 |
+
|
| 7 |
+
cpu_count="$(nproc 2>/dev/null || echo 8)"
|
| 8 |
+
default_jobs="$cpu_count"
|
| 9 |
+
if (( default_jobs > 8 )); then
|
| 10 |
+
default_jobs=8
|
| 11 |
+
fi
|
| 12 |
+
JOBS="${JOBS:-$default_jobs}"
|
| 13 |
+
|
| 14 |
+
usage() {
|
| 15 |
+
cat <<EOF
|
| 16 |
+
Usage: $(basename "$0") [-j JOBS] [-s SRC_DIR] [-d DST_DIR]
|
| 17 |
+
|
| 18 |
+
Options:
|
| 19 |
+
-j JOBS 并行解压进程数 (默认: min(nproc, 8))
|
| 20 |
+
-s SRC_DIR 压缩包目录 (默认: /cpfs01/videogen/gvhmr_res)
|
| 21 |
+
-d DST_DIR 解压目标目录 (默认: /cpfs01/videogen/gvhmr_smpl)
|
| 22 |
+
-h 显示帮助
|
| 23 |
+
EOF
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
while getopts ":j:s:d:h" opt; do
|
| 27 |
+
case "$opt" in
|
| 28 |
+
j) JOBS="$OPTARG" ;;
|
| 29 |
+
s) SRC_DIR="$OPTARG" ;;
|
| 30 |
+
d) DST_DIR="$OPTARG" ;;
|
| 31 |
+
h)
|
| 32 |
+
usage
|
| 33 |
+
exit 0
|
| 34 |
+
;;
|
| 35 |
+
:)
|
| 36 |
+
echo "Error: option -$OPTARG requires an argument." >&2
|
| 37 |
+
usage
|
| 38 |
+
exit 1
|
| 39 |
+
;;
|
| 40 |
+
\?)
|
| 41 |
+
echo "Error: invalid option -$OPTARG" >&2
|
| 42 |
+
usage
|
| 43 |
+
exit 1
|
| 44 |
+
;;
|
| 45 |
+
esac
|
| 46 |
+
done
|
| 47 |
+
|
| 48 |
+
if ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then
|
| 49 |
+
echo "Error: JOBS must be a positive integer, got: $JOBS" >&2
|
| 50 |
+
exit 1
|
| 51 |
+
fi
|
| 52 |
+
|
| 53 |
+
if [[ ! -d "$SRC_DIR" ]]; then
|
| 54 |
+
echo "Error: source directory not found: $SRC_DIR" >&2
|
| 55 |
+
exit 1
|
| 56 |
+
fi
|
| 57 |
+
|
| 58 |
+
mkdir -p "$DST_DIR"
|
| 59 |
+
|
| 60 |
+
strip_archive_ext() {
|
| 61 |
+
local name="$1"
|
| 62 |
+
case "$name" in
|
| 63 |
+
*.tar.gz) printf '%s' "${name%.tar.gz}" ;;
|
| 64 |
+
*.tgz) printf '%s' "${name%.tgz}" ;;
|
| 65 |
+
*.tar.xz) printf '%s' "${name%.tar.xz}" ;;
|
| 66 |
+
*.txz) printf '%s' "${name%.txz}" ;;
|
| 67 |
+
*.tar.bz2) printf '%s' "${name%.tar.bz2}" ;;
|
| 68 |
+
*.tbz2) printf '%s' "${name%.tbz2}" ;;
|
| 69 |
+
*.tar.zst) printf '%s' "${name%.tar.zst}" ;;
|
| 70 |
+
*.tzst) printf '%s' "${name%.tzst}" ;;
|
| 71 |
+
*.tar) printf '%s' "${name%.tar}" ;;
|
| 72 |
+
*) printf '%s' "$name" ;;
|
| 73 |
+
esac
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
render_progress() {
|
| 77 |
+
local done="$1"
|
| 78 |
+
local total="$2"
|
| 79 |
+
local width=40
|
| 80 |
+
local filled=$(( done * width / total ))
|
| 81 |
+
local empty=$(( width - filled ))
|
| 82 |
+
local percent=$(( done * 100 / total ))
|
| 83 |
+
local bar_fill bar_empty
|
| 84 |
+
|
| 85 |
+
printf -v bar_fill "%${filled}s" ""
|
| 86 |
+
bar_fill="${bar_fill// /#}"
|
| 87 |
+
printf -v bar_empty "%${empty}s" ""
|
| 88 |
+
bar_empty="${bar_empty// /-}"
|
| 89 |
+
|
| 90 |
+
printf "\r[%s%s] %d/%d (%3d%%)" "$bar_fill" "$bar_empty" "$done" "$total" "$percent"
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
extract_one() {
|
| 94 |
+
local tar_file="$1"
|
| 95 |
+
local rel stem out_dir
|
| 96 |
+
|
| 97 |
+
rel="${tar_file#"$SRC_DIR"/}"
|
| 98 |
+
if [[ "$rel" == "$tar_file" ]]; then
|
| 99 |
+
rel="$(basename "$tar_file")"
|
| 100 |
+
fi
|
| 101 |
+
|
| 102 |
+
stem="$(strip_archive_ext "$rel")"
|
| 103 |
+
out_dir="$DST_DIR/$stem"
|
| 104 |
+
|
| 105 |
+
mkdir -p "$out_dir"
|
| 106 |
+
tar -xf "$tar_file" -C "$out_dir"
|
| 107 |
+
printf '%s\n' "$tar_file"
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
export SRC_DIR DST_DIR
|
| 111 |
+
export -f strip_archive_ext extract_one
|
| 112 |
+
|
| 113 |
+
mapfile -d '' TAR_FILES < <(
|
| 114 |
+
find "$SRC_DIR" -type f \
|
| 115 |
+
\( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.xz' -o -name '*.txz' -o -name '*.tar.bz2' -o -name '*.tbz2' -o -name '*.tar.zst' -o -name '*.tzst' \) \
|
| 116 |
+
-print0 | sort -z
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
TOTAL="${#TAR_FILES[@]}"
|
| 120 |
+
if (( TOTAL == 0 )); then
|
| 121 |
+
echo "No tar archives found in: $SRC_DIR"
|
| 122 |
+
exit 0
|
| 123 |
+
fi
|
| 124 |
+
|
| 125 |
+
echo "Source: $SRC_DIR"
|
| 126 |
+
echo "Target: $DST_DIR"
|
| 127 |
+
echo "Archives: $TOTAL"
|
| 128 |
+
echo "Jobs: $JOBS"
|
| 129 |
+
|
| 130 |
+
start_ts="$(date +%s)"
|
| 131 |
+
done_count=0
|
| 132 |
+
render_progress 0 "$TOTAL"
|
| 133 |
+
|
| 134 |
+
printf '%s\0' "${TAR_FILES[@]}" \
|
| 135 |
+
| xargs -0 -n1 -P "$JOBS" bash -c 'extract_one "$1"' _ \
|
| 136 |
+
| while IFS= read -r _; do
|
| 137 |
+
done_count=$((done_count + 1))
|
| 138 |
+
render_progress "$done_count" "$TOTAL"
|
| 139 |
+
done
|
| 140 |
+
|
| 141 |
+
echo
|
| 142 |
+
end_ts="$(date +%s)"
|
| 143 |
+
elapsed="$((end_ts - start_ts))"
|
| 144 |
+
echo "Done in ${elapsed}s."
|
fast_rm.sh
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
usage() {
|
| 5 |
+
echo "Usage: bash $0 TARGET_DIR [JOBS]" >&2
|
| 6 |
+
exit 2
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
die() {
|
| 10 |
+
echo "fast_rm: $*" >&2
|
| 11 |
+
exit 2
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
format_bytes() {
|
| 15 |
+
awk -v bytes="${1:-0}" 'BEGIN {
|
| 16 |
+
split("B KiB MiB GiB TiB PiB", units, " ")
|
| 17 |
+
value = bytes + 0
|
| 18 |
+
unit = 1
|
| 19 |
+
while (value >= 1024 && unit < 6) {
|
| 20 |
+
value /= 1024
|
| 21 |
+
unit++
|
| 22 |
+
}
|
| 23 |
+
if (unit == 1) {
|
| 24 |
+
printf "%.0f %s", value, units[unit]
|
| 25 |
+
} else {
|
| 26 |
+
printf "%.2f %s", value, units[unit]
|
| 27 |
+
}
|
| 28 |
+
}'
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
fs_avail_bytes() {
|
| 32 |
+
df -B1 --output=avail "$1" 2>/dev/null | awk 'NR==2 {print $1}'
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
progress_bar() {
|
| 36 |
+
local freed=$1
|
| 37 |
+
local total=$2
|
| 38 |
+
local items=$3
|
| 39 |
+
local final=${4:-0}
|
| 40 |
+
local now elapsed percent filled empty bar rate capped
|
| 41 |
+
local width=30
|
| 42 |
+
|
| 43 |
+
now=$(date +%s)
|
| 44 |
+
elapsed=$((now - start_ts))
|
| 45 |
+
[ "$elapsed" -gt 0 ] || elapsed=1
|
| 46 |
+
[ "$freed" -ge 0 ] || freed=0
|
| 47 |
+
rate=$(awk -v b="$freed" -v s="$elapsed" 'BEGIN { if (s <= 0) s = 1; printf "%.0f", b / s }')
|
| 48 |
+
|
| 49 |
+
if [ "$total" -gt 0 ]; then
|
| 50 |
+
capped=$freed
|
| 51 |
+
[ "$capped" -le "$total" ] || capped=$total
|
| 52 |
+
percent=$((capped * 100 / total))
|
| 53 |
+
[ "$percent" -le 100 ] || percent=100
|
| 54 |
+
else
|
| 55 |
+
percent=0
|
| 56 |
+
fi
|
| 57 |
+
|
| 58 |
+
if [ "$total" -gt 0 ]; then
|
| 59 |
+
filled=$((percent * width / 100))
|
| 60 |
+
empty=$((width - filled))
|
| 61 |
+
bar=$(printf '%*s' "$filled" '' | tr ' ' '#')
|
| 62 |
+
bar=$bar$(printf '%*s' "$empty" '' | tr ' ' '-')
|
| 63 |
+
printf '\rfast_rm: [%s] %3d%% freed %s/%s rate %s/s items %d' \
|
| 64 |
+
"$bar" "$percent" "$(format_bytes "$freed")" "$(format_bytes "$total")" "$(format_bytes "$rate")" "$items" >&2
|
| 65 |
+
else
|
| 66 |
+
printf '\rfast_rm: freed %s rate %s/s items %d' \
|
| 67 |
+
"$(format_bytes "$freed")" "$(format_bytes "$rate")" "$items" >&2
|
| 68 |
+
fi
|
| 69 |
+
|
| 70 |
+
if [ "$final" = 1 ]; then
|
| 71 |
+
printf '\n' >&2
|
| 72 |
+
fi
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
monitor_progress() {
|
| 76 |
+
local total=$1
|
| 77 |
+
local watch_path=$2
|
| 78 |
+
local start_avail=$3
|
| 79 |
+
local current_avail=0
|
| 80 |
+
local freed=0
|
| 81 |
+
local max_freed=0
|
| 82 |
+
local items=0
|
| 83 |
+
local last_render=0
|
| 84 |
+
local now msg
|
| 85 |
+
|
| 86 |
+
while true; do
|
| 87 |
+
if IFS= read -r -t "$progress_interval" msg; then
|
| 88 |
+
[ "$msg" != "__FAST_RM_DONE__" ] || break
|
| 89 |
+
items=$((items + 1))
|
| 90 |
+
fi
|
| 91 |
+
|
| 92 |
+
current_avail=$(fs_avail_bytes "$watch_path")
|
| 93 |
+
case "$current_avail" in
|
| 94 |
+
''|*[!0-9]*) current_avail=$start_avail ;;
|
| 95 |
+
esac
|
| 96 |
+
freed=$((current_avail - start_avail))
|
| 97 |
+
[ "$freed" -ge 0 ] || freed=0
|
| 98 |
+
if [ "$freed" -gt "$max_freed" ]; then
|
| 99 |
+
max_freed=$freed
|
| 100 |
+
fi
|
| 101 |
+
|
| 102 |
+
now=$(date +%s)
|
| 103 |
+
if [ $((now - last_render)) -ge "$progress_interval" ]; then
|
| 104 |
+
progress_bar "$max_freed" "$total" "$items" 0
|
| 105 |
+
last_render=$now
|
| 106 |
+
fi
|
| 107 |
+
done
|
| 108 |
+
|
| 109 |
+
current_avail=$(fs_avail_bytes "$watch_path")
|
| 110 |
+
case "$current_avail" in
|
| 111 |
+
''|*[!0-9]*) current_avail=$start_avail ;;
|
| 112 |
+
esac
|
| 113 |
+
freed=$((current_avail - start_avail))
|
| 114 |
+
[ "$freed" -ge 0 ] || freed=0
|
| 115 |
+
if [ "$freed" -gt "$max_freed" ]; then
|
| 116 |
+
max_freed=$freed
|
| 117 |
+
fi
|
| 118 |
+
progress_bar "$max_freed" "$total" "$items" 1
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
delete_stage() {
|
| 122 |
+
local min_depth=$1
|
| 123 |
+
local max_depth=$2
|
| 124 |
+
|
| 125 |
+
if [ "$progress" = 1 ]; then
|
| 126 |
+
find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \
|
| 127 |
+
| FAST_RM_PROGRESS_FIFO=$progress_fifo xargs -0 -r -n 1 -P "$jobs" bash -c '
|
| 128 |
+
set -e
|
| 129 |
+
for path do
|
| 130 |
+
rm -rf -- "$path"
|
| 131 |
+
printf "1\n" > "$FAST_RM_PROGRESS_FIFO" || true
|
| 132 |
+
done
|
| 133 |
+
' _
|
| 134 |
+
else
|
| 135 |
+
find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \
|
| 136 |
+
| xargs -0 -r -n 1 -P "$jobs" rm -rf --
|
| 137 |
+
fi
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
[ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
|
| 141 |
+
|
| 142 |
+
target=$1
|
| 143 |
+
jobs=${2:-${FAST_RM_JOBS:-$(nproc 2>/dev/null || echo 32)}}
|
| 144 |
+
shard_depth=${FAST_RM_SHARD_DEPTH:-3}
|
| 145 |
+
progress=${FAST_RM_PROGRESS:-1}
|
| 146 |
+
progress_interval=${FAST_RM_PROGRESS_INTERVAL:-1}
|
| 147 |
+
exact_total=${FAST_RM_EXACT_TOTAL:-0}
|
| 148 |
+
|
| 149 |
+
case "$jobs" in
|
| 150 |
+
''|*[!0-9]*) die "JOBS must be a positive integer" ;;
|
| 151 |
+
esac
|
| 152 |
+
case "$shard_depth" in
|
| 153 |
+
''|*[!0-9]*) die "FAST_RM_SHARD_DEPTH must be a positive integer" ;;
|
| 154 |
+
esac
|
| 155 |
+
case "$progress" in
|
| 156 |
+
0|1) ;;
|
| 157 |
+
*) die "FAST_RM_PROGRESS must be 0 or 1" ;;
|
| 158 |
+
esac
|
| 159 |
+
case "$exact_total" in
|
| 160 |
+
0|1) ;;
|
| 161 |
+
*) die "FAST_RM_EXACT_TOTAL must be 0 or 1" ;;
|
| 162 |
+
esac
|
| 163 |
+
case "$progress_interval" in
|
| 164 |
+
''|*[!0-9]*) die "FAST_RM_PROGRESS_INTERVAL must be a positive integer" ;;
|
| 165 |
+
esac
|
| 166 |
+
|
| 167 |
+
[ "$jobs" -gt 0 ] || die "JOBS must be > 0"
|
| 168 |
+
[ "$shard_depth" -gt 0 ] || die "FAST_RM_SHARD_DEPTH must be > 0"
|
| 169 |
+
[ "$progress_interval" -gt 0 ] || die "FAST_RM_PROGRESS_INTERVAL must be > 0"
|
| 170 |
+
|
| 171 |
+
while [ "$target" != "/" ] && [ "${target%/}" != "$target" ]; do
|
| 172 |
+
target=${target%/}
|
| 173 |
+
done
|
| 174 |
+
|
| 175 |
+
[ -n "$target" ] || die "empty target"
|
| 176 |
+
[ "$target" != "/" ] || die "refuse to delete /"
|
| 177 |
+
|
| 178 |
+
parent=$(dirname -- "$target")
|
| 179 |
+
base=$(basename -- "$target")
|
| 180 |
+
[ "$base" != "." ] && [ "$base" != ".." ] || die "bad target: $target"
|
| 181 |
+
|
| 182 |
+
parent_abs=$(realpath -- "$parent")
|
| 183 |
+
target_abs=$parent_abs/$base
|
| 184 |
+
|
| 185 |
+
if [ ! -e "$target_abs" ] && [ ! -L "$target_abs" ]; then
|
| 186 |
+
echo "fast_rm: not found: $target_abs"
|
| 187 |
+
exit 0
|
| 188 |
+
fi
|
| 189 |
+
|
| 190 |
+
home_abs=$(realpath -- "${HOME:-/nonexistent}" 2>/dev/null || true)
|
| 191 |
+
case "$target_abs" in
|
| 192 |
+
"/"|"/cpfs01"|"/cpfs01/users"|"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong"|"/mnt"|"/mnt/workspace")
|
| 193 |
+
die "refuse to delete protected path: $target_abs"
|
| 194 |
+
;;
|
| 195 |
+
esac
|
| 196 |
+
[ -z "$home_abs" ] || [ "$target_abs" != "$home_abs" ] || die "refuse to delete HOME: $target_abs"
|
| 197 |
+
|
| 198 |
+
mount_point=$(findmnt -T "$target_abs" -n -o TARGET 2>/dev/null || true)
|
| 199 |
+
[ -z "$mount_point" ] || [ "$mount_point" != "$target_abs" ] || die "refuse to delete mount point: $target_abs"
|
| 200 |
+
|
| 201 |
+
cwd=$(pwd -P)
|
| 202 |
+
case "$cwd/" in
|
| 203 |
+
"$target_abs"/*|"$target_abs/"*) die "refuse to delete current working tree: $target_abs" ;;
|
| 204 |
+
esac
|
| 205 |
+
|
| 206 |
+
if [ ! -d "$target_abs" ] || [ -L "$target_abs" ]; then
|
| 207 |
+
rm -f -- "$target_abs"
|
| 208 |
+
exit 0
|
| 209 |
+
fi
|
| 210 |
+
|
| 211 |
+
progress_fifo=
|
| 212 |
+
progress_pid=
|
| 213 |
+
progress_keepalive_fd=
|
| 214 |
+
cleanup_progress() {
|
| 215 |
+
if [ -n "$progress_pid" ]; then
|
| 216 |
+
kill "$progress_pid" 2>/dev/null || true
|
| 217 |
+
fi
|
| 218 |
+
if [ -n "$progress_keepalive_fd" ]; then
|
| 219 |
+
exec {progress_keepalive_fd}>&- 2>/dev/null || true
|
| 220 |
+
fi
|
| 221 |
+
if [ -n "$progress_fifo" ]; then
|
| 222 |
+
rm -f -- "$progress_fifo"
|
| 223 |
+
fi
|
| 224 |
+
}
|
| 225 |
+
trap cleanup_progress EXIT
|
| 226 |
+
|
| 227 |
+
safe_base=$(printf '%s' "$base" | tr -c 'A-Za-z0-9._-' '_')
|
| 228 |
+
if [[ "$base" == .deleting_* ]]; then
|
| 229 |
+
trash=$target_abs
|
| 230 |
+
else
|
| 231 |
+
trash=$parent_abs/.deleting_${safe_base}.$(date +%Y%m%d_%H%M%S).$$
|
| 232 |
+
mv -- "$target_abs" "$trash"
|
| 233 |
+
fi
|
| 234 |
+
|
| 235 |
+
echo "fast_rm: deleting $trash with jobs=$jobs shard_depth=$shard_depth"
|
| 236 |
+
|
| 237 |
+
if [ "$progress" = 1 ]; then
|
| 238 |
+
total_bytes=0
|
| 239 |
+
if [ "$exact_total" = 1 ]; then
|
| 240 |
+
echo "fast_rm: exact sizing target..." >&2
|
| 241 |
+
total_bytes=$(du -sB1 -- "$trash" 2>/dev/null | awk 'NR==1 {print $1}')
|
| 242 |
+
case "$total_bytes" in
|
| 243 |
+
''|*[!0-9]*) total_bytes=0 ;;
|
| 244 |
+
esac
|
| 245 |
+
echo "fast_rm: total $(format_bytes "$total_bytes")" >&2
|
| 246 |
+
else
|
| 247 |
+
echo "fast_rm: skip exact sizing; showing freed space estimate" >&2
|
| 248 |
+
fi
|
| 249 |
+
start_avail=$(fs_avail_bytes "$parent_abs")
|
| 250 |
+
case "$start_avail" in
|
| 251 |
+
''|*[!0-9]*) start_avail=0 ;;
|
| 252 |
+
esac
|
| 253 |
+
progress_fifo=$(mktemp -u "${TMPDIR:-/tmp}/fast_rm_progress.XXXXXX")
|
| 254 |
+
mkfifo "$progress_fifo"
|
| 255 |
+
start_ts=$(date +%s)
|
| 256 |
+
monitor_progress "$total_bytes" "$parent_abs" "$start_avail" < "$progress_fifo" &
|
| 257 |
+
progress_pid=$!
|
| 258 |
+
exec {progress_keepalive_fd}>"$progress_fifo"
|
| 259 |
+
fi
|
| 260 |
+
|
| 261 |
+
if [ "$shard_depth" -gt 1 ]; then
|
| 262 |
+
delete_stage "$shard_depth" "$shard_depth"
|
| 263 |
+
fi
|
| 264 |
+
|
| 265 |
+
delete_stage 1 1
|
| 266 |
+
|
| 267 |
+
rmdir -- "$trash" 2>/dev/null || rm -rf -- "$trash"
|
| 268 |
+
|
| 269 |
+
if [ "$progress" = 1 ]; then
|
| 270 |
+
printf '%s\n' "__FAST_RM_DONE__" >&$progress_keepalive_fd || true
|
| 271 |
+
wait "$progress_pid" 2>/dev/null || true
|
| 272 |
+
progress_pid=
|
| 273 |
+
exec {progress_keepalive_fd}>&- 2>/dev/null || true
|
| 274 |
+
progress_keepalive_fd=
|
| 275 |
+
rm -f -- "$progress_fifo"
|
| 276 |
+
progress_fifo=
|
| 277 |
+
fi
|
| 278 |
+
|
| 279 |
+
echo "fast_rm: done"
|
oss_download.sh
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 用cp只有几m每秒,特别慢,但是用api就特别快
|
| 2 |
+
# TODO: 封装成一个下载目录的脚本
|
| 3 |
+
|
| 4 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 5 |
+
# ./ossutil cp -r \
|
| 6 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/v11-20260202-204134/epoch=4-step=1600.ckpt \
|
| 7 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/v11-20260202-204134 \
|
| 8 |
+
# --parallel 32
|
| 9 |
+
|
| 10 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 11 |
+
# ./ossutil cp -r \
|
| 12 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/vae-20260123-085324/epoch=99-step=34000.ckpt \
|
| 13 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/vae-20260123-085324 \
|
| 14 |
+
# --parallel 32
|
| 15 |
+
|
| 16 |
+
# 这样下载好像有问题,会在文件末尾写入类似5.546607(s) elapsed的字符
|
| 17 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 18 |
+
# ./ossutil api get-object \
|
| 19 |
+
# --bucket cpfs-to-new-cpfs \
|
| 20 |
+
# --key VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/v11-20260202-204134/epoch=4-step=1600.ckpt/output_dir/model-00013-of-00013.safetensors \
|
| 21 |
+
# > /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/v11-20260202-204134/epoch=4-step=1600.ckpt/output_dir/model-00013-of-00013.safetensors
|
| 22 |
+
|
| 23 |
+
BUCKET="cpfs-to-new-cpfs"
|
| 24 |
+
OSS_PREFIX="VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/v15-20260224-013149-step3500-ae/"
|
| 25 |
+
LOCAL_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/v15-20260224-013149-step3500-ae/"
|
| 26 |
+
|
| 27 |
+
mkdir -p "$LOCAL_DIR"
|
| 28 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 29 |
+
|
| 30 |
+
./ossutil -c .ossutilconfig ls "oss://$BUCKET/$OSS_PREFIX" | grep -o "oss://$BUCKET/.*" | grep -v "/$" | while read -r OSS_URI; do
|
| 31 |
+
KEY="${OSS_URI#oss://$BUCKET/}"
|
| 32 |
+
RELATIVE_PATH="${KEY#$OSS_PREFIX}"
|
| 33 |
+
LOCAL_FILE="$LOCAL_DIR$RELATIVE_PATH"
|
| 34 |
+
LOCAL_SUBDIR=$(dirname "$LOCAL_FILE")
|
| 35 |
+
|
| 36 |
+
if [ ! -d "$LOCAL_SUBDIR" ]; then
|
| 37 |
+
mkdir -p "$LOCAL_SUBDIR"
|
| 38 |
+
fi
|
| 39 |
+
|
| 40 |
+
echo "downloading: $RELATIVE_PATH"
|
| 41 |
+
|
| 42 |
+
./ossutil -c .ossutilconfig api get-object \
|
| 43 |
+
--bucket "$BUCKET" \
|
| 44 |
+
--key "$KEY" \
|
| 45 |
+
> "$LOCAL_FILE"
|
| 46 |
+
|
| 47 |
+
if [ $? -eq 0 ]; then
|
| 48 |
+
python3 -c "
|
| 49 |
+
import sys, re
|
| 50 |
+
with open(sys.argv[1], 'r+b') as f:
|
| 51 |
+
f.seek(0, 2)
|
| 52 |
+
size = f.tell()
|
| 53 |
+
if size == 0:
|
| 54 |
+
sys.exit(0)
|
| 55 |
+
chunk = min(4096, size)
|
| 56 |
+
f.seek(-chunk, 2)
|
| 57 |
+
tail = f.read()
|
| 58 |
+
m = re.search(br'(?:\s*[0-9]+\.[0-9]+\(s\) elapsed\s*)+$', tail)
|
| 59 |
+
if m:
|
| 60 |
+
f.truncate(size - len(m.group(0)))
|
| 61 |
+
" "$LOCAL_FILE"
|
| 62 |
+
echo "成功 -> $RELATIVE_PATH"
|
| 63 |
+
else
|
| 64 |
+
echo "失败 -> $KEY"
|
| 65 |
+
echo "失败 -> $RELATIVE_PATH"
|
| 66 |
+
fi
|
| 67 |
+
done
|
oss_upload.sh
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 2 |
+
# ./ossutil cp -r \
|
| 3 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/v6x/ \
|
| 4 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan
|
| 5 |
+
|
| 6 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 7 |
+
# ./ossutil cp -r \
|
| 8 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/2026011x/ \
|
| 9 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan
|
| 10 |
+
|
| 11 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 12 |
+
# ./ossutil cp -r \
|
| 13 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/loading_oss/ \
|
| 14 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan
|
| 15 |
+
|
| 16 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/ossutil-2.2.0-linux-amd64
|
| 17 |
+
./ossutil -c .ossutilconfig \
|
| 18 |
+
cp -r \
|
| 19 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/datasets/HumanML3D_token \
|
| 20 |
+
oss://cpfs-to-new-cpfs/VideoGen/user_data/yetinghong/datasets/HumanML3D_token
|
| 21 |
+
|
| 22 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 23 |
+
# ./ossutil cp -r \
|
| 24 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_2858_pose_embed_v2/ \
|
| 25 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/dance_2858_pose_embed_v2
|
| 26 |
+
|
| 27 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
|
| 28 |
+
# ./ossutil cp -r \
|
| 29 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/dance_1723_new/ \
|
| 30 |
+
# oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/dance_1723_new
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
|
read_pt.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
# 1. 你的文件路径
|
| 4 |
+
file_path = "/cpfs01/videogen/gvhmr_smpl/part_10/--9DvmkocUk_00010_44_171/hmr4d_results.pt"
|
| 5 |
+
|
| 6 |
+
# 2. 加载数据
|
| 7 |
+
data = torch.load(file_path)
|
| 8 |
+
|
| 9 |
+
# 3. 指定要检查的四个 key
|
| 10 |
+
keys_to_check = ['smpl_params_global', 'smpl_params_incam', 'K_fullimg', 'net_outputs']
|
| 11 |
+
|
| 12 |
+
print(f"正在检查文件结构...")
|
| 13 |
+
print("=" * 50)
|
| 14 |
+
|
| 15 |
+
# 4. 开始遍历检查
|
| 16 |
+
for key in keys_to_check:
|
| 17 |
+
if key not in data:
|
| 18 |
+
print(f"❌ 找不到: {key}")
|
| 19 |
+
continue
|
| 20 |
+
|
| 21 |
+
val = data[key]
|
| 22 |
+
|
| 23 |
+
# 情况 A:直接是张量 (例如 K_fullimg)
|
| 24 |
+
if isinstance(val, torch.Tensor):
|
| 25 |
+
print(f"✅ 【{key}】 (直接是张量) \n └── 形状: {val.shape}")
|
| 26 |
+
|
| 27 |
+
# 情况 B:是嵌套字典 (例如 smpl_params_global, net_outputs)
|
| 28 |
+
elif isinstance(val, dict):
|
| 29 |
+
print(f"📦 【{key}】 (嵌套字典) 里面包含:")
|
| 30 |
+
for sub_k, sub_v in val.items():
|
| 31 |
+
if isinstance(sub_v, torch.Tensor):
|
| 32 |
+
print(f" ├── {sub_k}: {sub_v.shape}")
|
| 33 |
+
elif isinstance(sub_v, dict):
|
| 34 |
+
print(f" ├── {sub_k}: [又是一个字典] 包含 keys -> {list(sub_v.keys())}")
|
| 35 |
+
else:
|
| 36 |
+
print(f" ├── {sub_k}: [其他类型] {type(sub_v)}")
|
| 37 |
+
|
| 38 |
+
# 情况 C:其他未知类型
|
| 39 |
+
else:
|
| 40 |
+
print(f"❓ 【{key}】 (其他类型: {type(val)})")
|
| 41 |
+
|
| 42 |
+
print("-" * 50)
|
repair_cache_jsonl.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Repair cached confidence/reason fields in JSONL files in place."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from collections import Counter
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from tempfile import NamedTemporaryFile
|
| 12 |
+
from typing import Dict, Iterable, Iterator, List, Tuple
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 15 |
+
if str(ROOT) not in sys.path:
|
| 16 |
+
sys.path.insert(0, str(ROOT))
|
| 17 |
+
|
| 18 |
+
from answer_utils import repair_confidence_reason_record_fields
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def parse_args() -> argparse.Namespace:
|
| 22 |
+
parser = argparse.ArgumentParser(
|
| 23 |
+
description="Repair cache JSONL files by backfilling conf/reason from raw output."
|
| 24 |
+
)
|
| 25 |
+
parser.add_argument(
|
| 26 |
+
"paths",
|
| 27 |
+
nargs="*",
|
| 28 |
+
default=["cache/benchmarks"],
|
| 29 |
+
help="Files or directories to scan. Defaults to cache/benchmarks.",
|
| 30 |
+
)
|
| 31 |
+
parser.add_argument(
|
| 32 |
+
"--dry-run",
|
| 33 |
+
action="store_true",
|
| 34 |
+
help="Scan and report changes without rewriting files.",
|
| 35 |
+
)
|
| 36 |
+
parser.add_argument(
|
| 37 |
+
"--verbose",
|
| 38 |
+
action="store_true",
|
| 39 |
+
help="Print per-file repair counts.",
|
| 40 |
+
)
|
| 41 |
+
return parser.parse_args()
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def iter_jsonl_files(paths: Iterable[str]) -> Iterator[Path]:
|
| 45 |
+
seen = set()
|
| 46 |
+
for raw_path in paths:
|
| 47 |
+
path = Path(raw_path)
|
| 48 |
+
if path.is_file() and path.suffix == ".jsonl":
|
| 49 |
+
resolved = path.resolve()
|
| 50 |
+
if resolved not in seen:
|
| 51 |
+
seen.add(resolved)
|
| 52 |
+
yield path
|
| 53 |
+
continue
|
| 54 |
+
if path.is_dir():
|
| 55 |
+
for item in sorted(path.rglob("*.jsonl")):
|
| 56 |
+
resolved = item.resolve()
|
| 57 |
+
if resolved in seen:
|
| 58 |
+
continue
|
| 59 |
+
seen.add(resolved)
|
| 60 |
+
yield item
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def changed_field_counts(before: Dict, after: Dict) -> Counter:
|
| 64 |
+
counts: Counter = Counter()
|
| 65 |
+
keys = set(before.keys()) | set(after.keys())
|
| 66 |
+
for key in keys:
|
| 67 |
+
if before.get(key) == after.get(key):
|
| 68 |
+
continue
|
| 69 |
+
if key == "conf" or key.endswith("_conf"):
|
| 70 |
+
counts["conf_fields"] += 1
|
| 71 |
+
elif key == "reason" or key.endswith("_reason"):
|
| 72 |
+
counts["reason_fields"] += 1
|
| 73 |
+
else:
|
| 74 |
+
counts["other_fields"] += 1
|
| 75 |
+
return counts
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def repair_file(path: Path, dry_run: bool = False) -> Tuple[bool, Counter]:
|
| 79 |
+
file_counts: Counter = Counter()
|
| 80 |
+
repaired_lines: List[str] = []
|
| 81 |
+
|
| 82 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 83 |
+
for line_no, raw_line in enumerate(handle, start=1):
|
| 84 |
+
stripped = raw_line.strip()
|
| 85 |
+
if not stripped:
|
| 86 |
+
repaired_lines.append(raw_line)
|
| 87 |
+
continue
|
| 88 |
+
try:
|
| 89 |
+
record = json.loads(stripped)
|
| 90 |
+
except json.JSONDecodeError as exc:
|
| 91 |
+
raise ValueError(f"{path}:{line_no}: invalid json: {exc}") from exc
|
| 92 |
+
|
| 93 |
+
repaired = repair_confidence_reason_record_fields(record)
|
| 94 |
+
if repaired != record:
|
| 95 |
+
file_counts["records_changed"] += 1
|
| 96 |
+
file_counts.update(changed_field_counts(record, repaired))
|
| 97 |
+
repaired_lines.append(json.dumps(repaired, ensure_ascii=False) + "\n")
|
| 98 |
+
|
| 99 |
+
changed = bool(file_counts["records_changed"])
|
| 100 |
+
if changed and not dry_run:
|
| 101 |
+
with NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
|
| 102 |
+
tmp.writelines(repaired_lines)
|
| 103 |
+
tmp_path = Path(tmp.name)
|
| 104 |
+
tmp_path.replace(path)
|
| 105 |
+
|
| 106 |
+
return changed, file_counts
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def main() -> int:
|
| 110 |
+
args = parse_args()
|
| 111 |
+
totals: Counter = Counter()
|
| 112 |
+
files = list(iter_jsonl_files(args.paths))
|
| 113 |
+
|
| 114 |
+
if not files:
|
| 115 |
+
print("No .jsonl files found.")
|
| 116 |
+
return 0
|
| 117 |
+
|
| 118 |
+
for path in files:
|
| 119 |
+
changed, file_counts = repair_file(path, dry_run=args.dry_run)
|
| 120 |
+
totals["files_scanned"] += 1
|
| 121 |
+
if changed:
|
| 122 |
+
totals["files_changed"] += 1
|
| 123 |
+
totals.update(file_counts)
|
| 124 |
+
if args.verbose and changed:
|
| 125 |
+
rel = path.as_posix()
|
| 126 |
+
print(
|
| 127 |
+
f"{rel}: records={file_counts['records_changed']} "
|
| 128 |
+
f"conf={file_counts['conf_fields']} reason={file_counts['reason_fields']}"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
mode = "dry-run" if args.dry_run else "rewritten"
|
| 132 |
+
print(
|
| 133 |
+
f"[{mode}] files_scanned={totals['files_scanned']} "
|
| 134 |
+
f"files_changed={totals['files_changed']} "
|
| 135 |
+
f"records_changed={totals['records_changed']} "
|
| 136 |
+
f"conf_fields={totals['conf_fields']} "
|
| 137 |
+
f"reason_fields={totals['reason_fields']}"
|
| 138 |
+
)
|
| 139 |
+
return 0
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
if __name__ == "__main__":
|
| 143 |
+
raise SystemExit(main())
|
repair_math_cache_answers.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Repair math benchmark cache predictions/correct flags in place using current answer parsing."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from collections import Counter, defaultdict
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from tempfile import NamedTemporaryFile
|
| 12 |
+
from typing import Any, Dict, Iterable, Iterator, List, Tuple
|
| 13 |
+
|
| 14 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 15 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 16 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 17 |
+
|
| 18 |
+
from answer_utils import (
|
| 19 |
+
check_answer,
|
| 20 |
+
extract_answer,
|
| 21 |
+
extract_confidence_reason,
|
| 22 |
+
repair_confidence_reason_record_fields,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def parse_args() -> argparse.Namespace:
|
| 27 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 28 |
+
parser.add_argument(
|
| 29 |
+
"--root",
|
| 30 |
+
default="cache/benchmarks/math",
|
| 31 |
+
help="Math benchmark cache root to repair. Default: cache/benchmarks/math",
|
| 32 |
+
)
|
| 33 |
+
parser.add_argument(
|
| 34 |
+
"--dry-run",
|
| 35 |
+
action="store_true",
|
| 36 |
+
help="Report intended changes without rewriting files.",
|
| 37 |
+
)
|
| 38 |
+
parser.add_argument(
|
| 39 |
+
"--verbose",
|
| 40 |
+
action="store_true",
|
| 41 |
+
help="Print per-file change counts.",
|
| 42 |
+
)
|
| 43 |
+
return parser.parse_args()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _safe_str(value: Any) -> str:
|
| 47 |
+
if value is None:
|
| 48 |
+
return ""
|
| 49 |
+
return str(value).strip()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _iter_stage_files(root: Path) -> Iterator[Path]:
|
| 53 |
+
for path in sorted(root.rglob("*.stage*.cached.jsonl")):
|
| 54 |
+
if path.is_file():
|
| 55 |
+
yield path
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _iter_only_files(root: Path) -> Iterator[Path]:
|
| 59 |
+
for path in sorted(root.rglob("only/*/*.jsonl")):
|
| 60 |
+
if path.is_file():
|
| 61 |
+
yield path
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _write_lines_atomic(path: Path, lines: List[str]) -> None:
|
| 65 |
+
with NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
|
| 66 |
+
tmp.writelines(lines)
|
| 67 |
+
tmp_path = Path(tmp.name)
|
| 68 |
+
tmp_path.replace(path)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _repair_stage_record(record: Dict[str, Any], counts: Counter) -> Dict[str, Any]:
|
| 72 |
+
updated = repair_confidence_reason_record_fields(record)
|
| 73 |
+
task = _safe_str(updated.get("task")).lower()
|
| 74 |
+
if task != "math":
|
| 75 |
+
return updated
|
| 76 |
+
|
| 77 |
+
gold = updated.get("label")
|
| 78 |
+
for stage_idx in (1, 2, 3):
|
| 79 |
+
raw_key = f"stage{stage_idx}_raw"
|
| 80 |
+
pred_key = f"stage{stage_idx}_pred"
|
| 81 |
+
correct_key = f"stage{stage_idx}_correct"
|
| 82 |
+
raw_text = _safe_str(updated.get(raw_key))
|
| 83 |
+
if raw_text:
|
| 84 |
+
new_pred = _safe_str(extract_answer(task, raw_text))
|
| 85 |
+
if new_pred != _safe_str(updated.get(pred_key)):
|
| 86 |
+
updated[pred_key] = new_pred
|
| 87 |
+
counts[f"stage{stage_idx}_pred_changed"] += 1
|
| 88 |
+
new_correct = bool(check_answer(task, new_pred, gold))
|
| 89 |
+
if bool(updated.get(correct_key, False)) != new_correct:
|
| 90 |
+
updated[correct_key] = new_correct
|
| 91 |
+
counts[f"stage{stage_idx}_correct_changed"] += 1
|
| 92 |
+
|
| 93 |
+
if stage_idx >= 3:
|
| 94 |
+
continue
|
| 95 |
+
retry_raw_key = f"stage{stage_idx}_retry_raw"
|
| 96 |
+
retry_pred_key = f"stage{stage_idx}_retry_pred"
|
| 97 |
+
retry_correct_key = f"stage{stage_idx}_retry_correct"
|
| 98 |
+
retry_raw = _safe_str(updated.get(retry_raw_key))
|
| 99 |
+
if retry_raw:
|
| 100 |
+
new_retry_pred = _safe_str(extract_answer(task, retry_raw))
|
| 101 |
+
if new_retry_pred != _safe_str(updated.get(retry_pred_key)):
|
| 102 |
+
updated[retry_pred_key] = new_retry_pred
|
| 103 |
+
counts[f"stage{stage_idx}_retry_pred_changed"] += 1
|
| 104 |
+
new_retry_correct = bool(check_answer(task, new_retry_pred, gold))
|
| 105 |
+
if bool(updated.get(retry_correct_key, False)) != new_retry_correct:
|
| 106 |
+
updated[retry_correct_key] = new_retry_correct
|
| 107 |
+
counts[f"stage{stage_idx}_retry_correct_changed"] += 1
|
| 108 |
+
return updated
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def repair_stage_file(path: Path, dry_run: bool = False) -> Counter:
|
| 112 |
+
counts: Counter = Counter()
|
| 113 |
+
out_lines: List[str] = []
|
| 114 |
+
changed = False
|
| 115 |
+
|
| 116 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 117 |
+
for raw_line in handle:
|
| 118 |
+
stripped = raw_line.strip()
|
| 119 |
+
if not stripped:
|
| 120 |
+
out_lines.append(raw_line)
|
| 121 |
+
continue
|
| 122 |
+
record = json.loads(stripped)
|
| 123 |
+
updated = _repair_stage_record(record, counts)
|
| 124 |
+
if updated != record:
|
| 125 |
+
changed = True
|
| 126 |
+
counts["records_changed"] += 1
|
| 127 |
+
out_lines.append(json.dumps(updated, ensure_ascii=False) + "\n")
|
| 128 |
+
|
| 129 |
+
if changed and not dry_run:
|
| 130 |
+
_write_lines_atomic(path, out_lines)
|
| 131 |
+
if changed:
|
| 132 |
+
counts["files_changed"] += 1
|
| 133 |
+
counts["files_scanned"] += 1
|
| 134 |
+
return counts
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _only_report_for_rows(path: Path, rows: List[Dict[str, Any]]) -> Dict[str, Any]:
|
| 138 |
+
total = len(rows)
|
| 139 |
+
correct = sum(int(bool(row.get("correct", False))) for row in rows)
|
| 140 |
+
report_path = path.with_suffix(".report.json")
|
| 141 |
+
existing = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
|
| 142 |
+
report = dict(existing)
|
| 143 |
+
report.update(
|
| 144 |
+
{
|
| 145 |
+
"path": str(path),
|
| 146 |
+
"split": path.stem,
|
| 147 |
+
"total": total,
|
| 148 |
+
"correct": correct,
|
| 149 |
+
"accuracy": (correct / total) if total else 0.0,
|
| 150 |
+
}
|
| 151 |
+
)
|
| 152 |
+
by_task = defaultdict(lambda: {"total": 0, "correct": 0})
|
| 153 |
+
for row in rows:
|
| 154 |
+
task = _safe_str(row.get("task"))
|
| 155 |
+
by_task[task]["total"] += 1
|
| 156 |
+
by_task[task]["correct"] += int(bool(row.get("correct", False)))
|
| 157 |
+
if existing.get("by_task"):
|
| 158 |
+
merged = {}
|
| 159 |
+
for task_name, stats in dict(existing["by_task"]).items():
|
| 160 |
+
merged[task_name] = dict(stats)
|
| 161 |
+
for task_name, stats in by_task.items():
|
| 162 |
+
merged.setdefault(task_name, {})
|
| 163 |
+
merged[task_name]["total"] = stats["total"]
|
| 164 |
+
merged[task_name]["correct"] = stats["correct"]
|
| 165 |
+
report["by_task"] = merged
|
| 166 |
+
else:
|
| 167 |
+
report["by_task"] = dict(by_task)
|
| 168 |
+
return report
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _repair_only_record(record: Dict[str, Any], counts: Counter) -> Dict[str, Any]:
|
| 172 |
+
updated = repair_confidence_reason_record_fields(record)
|
| 173 |
+
task = _safe_str(updated.get("task")).lower()
|
| 174 |
+
if task != "math":
|
| 175 |
+
return updated
|
| 176 |
+
|
| 177 |
+
raw_text = _safe_str(updated.get("raw"))
|
| 178 |
+
if raw_text:
|
| 179 |
+
new_pred = _safe_str(extract_answer(task, raw_text))
|
| 180 |
+
if new_pred != _safe_str(updated.get("pred")):
|
| 181 |
+
updated["pred"] = new_pred
|
| 182 |
+
counts["pred_changed"] += 1
|
| 183 |
+
conf, reason = extract_confidence_reason(raw_text)
|
| 184 |
+
if float(updated.get("conf", 0.0) or 0.0) != float(conf):
|
| 185 |
+
updated["conf"] = float(conf)
|
| 186 |
+
counts["conf_changed"] += 1
|
| 187 |
+
if _safe_str(updated.get("reason")) != _safe_str(reason):
|
| 188 |
+
updated["reason"] = _safe_str(reason)
|
| 189 |
+
counts["reason_changed"] += 1
|
| 190 |
+
new_correct = bool(check_answer(task, new_pred, updated.get("label")))
|
| 191 |
+
if bool(updated.get("correct", False)) != new_correct:
|
| 192 |
+
updated["correct"] = new_correct
|
| 193 |
+
counts["correct_changed"] += 1
|
| 194 |
+
return updated
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def repair_only_file(path: Path, dry_run: bool = False) -> Counter:
|
| 198 |
+
counts: Counter = Counter()
|
| 199 |
+
out_lines: List[str] = []
|
| 200 |
+
changed = False
|
| 201 |
+
rows: List[Dict[str, Any]] = []
|
| 202 |
+
|
| 203 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 204 |
+
for raw_line in handle:
|
| 205 |
+
stripped = raw_line.strip()
|
| 206 |
+
if not stripped:
|
| 207 |
+
out_lines.append(raw_line)
|
| 208 |
+
continue
|
| 209 |
+
record = json.loads(stripped)
|
| 210 |
+
updated = _repair_only_record(record, counts)
|
| 211 |
+
rows.append(updated)
|
| 212 |
+
if updated != record:
|
| 213 |
+
changed = True
|
| 214 |
+
counts["records_changed"] += 1
|
| 215 |
+
out_lines.append(json.dumps(updated, ensure_ascii=False) + "\n")
|
| 216 |
+
|
| 217 |
+
report = _only_report_for_rows(path, rows)
|
| 218 |
+
report_path = path.with_suffix(".report.json")
|
| 219 |
+
old_report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else None
|
| 220 |
+
report_changed = old_report != report
|
| 221 |
+
|
| 222 |
+
if (changed or report_changed) and not dry_run:
|
| 223 |
+
if changed:
|
| 224 |
+
_write_lines_atomic(path, out_lines)
|
| 225 |
+
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 226 |
+
|
| 227 |
+
if changed:
|
| 228 |
+
counts["files_changed"] += 1
|
| 229 |
+
if report_changed:
|
| 230 |
+
counts["reports_changed"] += 1
|
| 231 |
+
counts["files_scanned"] += 1
|
| 232 |
+
return counts
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def main() -> int:
|
| 236 |
+
args = parse_args()
|
| 237 |
+
root = Path(args.root)
|
| 238 |
+
if not root.exists():
|
| 239 |
+
raise FileNotFoundError(f"Root does not exist: {root}")
|
| 240 |
+
|
| 241 |
+
totals: Counter = Counter()
|
| 242 |
+
|
| 243 |
+
for path in _iter_stage_files(root):
|
| 244 |
+
file_counts = repair_stage_file(path, dry_run=args.dry_run)
|
| 245 |
+
totals.update(file_counts)
|
| 246 |
+
if args.verbose and (file_counts["records_changed"] or file_counts["files_changed"]):
|
| 247 |
+
print(f"{path}: stage_records_changed={file_counts['records_changed']}")
|
| 248 |
+
|
| 249 |
+
for path in _iter_only_files(root):
|
| 250 |
+
file_counts = repair_only_file(path, dry_run=args.dry_run)
|
| 251 |
+
totals.update(file_counts)
|
| 252 |
+
if args.verbose and (file_counts["records_changed"] or file_counts["reports_changed"]):
|
| 253 |
+
print(
|
| 254 |
+
f"{path}: only_records_changed={file_counts['records_changed']} "
|
| 255 |
+
f"reports_changed={file_counts['reports_changed']}"
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
mode = "dry-run" if args.dry_run else "rewritten"
|
| 259 |
+
print(
|
| 260 |
+
f"[{mode}] files_scanned={totals['files_scanned']} files_changed={totals['files_changed']} "
|
| 261 |
+
f"reports_changed={totals['reports_changed']} records_changed={totals['records_changed']}"
|
| 262 |
+
)
|
| 263 |
+
for key in sorted(k for k in totals if k not in {"files_scanned", "files_changed", "reports_changed", "records_changed"}):
|
| 264 |
+
print(f"{key}={totals[key]}")
|
| 265 |
+
return 0
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
if __name__ == "__main__":
|
| 269 |
+
raise SystemExit(main())
|
report_stage_costs.py
ADDED
|
@@ -0,0 +1,622 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Report per-stage cascade/only cost and accuracy statistics from benchmark caches."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
|
| 12 |
+
|
| 13 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 14 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 15 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 16 |
+
|
| 17 |
+
from transformers import AutoTokenizer
|
| 18 |
+
|
| 19 |
+
from answer_utils import check_answer, repair_confidence_reason_record_fields, task_instruction
|
| 20 |
+
from cascade_utils import (
|
| 21 |
+
build_router_prompt_for_cost,
|
| 22 |
+
count_tokens,
|
| 23 |
+
iter_stage_indices,
|
| 24 |
+
render_chat,
|
| 25 |
+
safe_str,
|
| 26 |
+
stage_correct_key,
|
| 27 |
+
stage_output_exists,
|
| 28 |
+
stage_pred_key,
|
| 29 |
+
stage_raw_key,
|
| 30 |
+
stage_retry_correct_key,
|
| 31 |
+
stage_retry_output_exists,
|
| 32 |
+
stage_retry_pred_key,
|
| 33 |
+
stage_retry_raw_key,
|
| 34 |
+
)
|
| 35 |
+
from config import MODEL_GROUPS, SYSTEM_MESSAGE_DEFAULT
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _read_jsonl(path: Path) -> List[Dict]:
|
| 39 |
+
rows: List[Dict] = []
|
| 40 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 41 |
+
for line in handle:
|
| 42 |
+
line = line.strip()
|
| 43 |
+
if line:
|
| 44 |
+
rows.append(repair_confidence_reason_record_fields(json.loads(line)))
|
| 45 |
+
return rows
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _price_per_token(price_per_million: float) -> float:
|
| 49 |
+
return float(price_per_million) / 1_000_000.0
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _parse_tokenizer_overrides(values: Sequence[str]) -> Dict[Tuple[str, str], str]:
|
| 53 |
+
overrides: Dict[Tuple[str, str], str] = {}
|
| 54 |
+
for value in values:
|
| 55 |
+
raw = safe_str(value)
|
| 56 |
+
if not raw:
|
| 57 |
+
continue
|
| 58 |
+
if "=" not in raw or ":" not in raw.split("=", 1)[0]:
|
| 59 |
+
raise ValueError(
|
| 60 |
+
f"Invalid --tokenizer-source {value!r}. Expected format: <group>:<stage>=<source>"
|
| 61 |
+
)
|
| 62 |
+
key_part, source = raw.split("=", 1)
|
| 63 |
+
group, stage = key_part.split(":", 1)
|
| 64 |
+
group = safe_str(group).lower()
|
| 65 |
+
stage = safe_str(stage).lower()
|
| 66 |
+
source = safe_str(source)
|
| 67 |
+
if not group or not stage or not source:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
f"Invalid --tokenizer-source {value!r}. Expected format: <group>:<stage>=<source>"
|
| 70 |
+
)
|
| 71 |
+
overrides[(group, stage)] = source
|
| 72 |
+
return overrides
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _resolve_tokenizer_source(
|
| 76 |
+
stage_cfg: Dict,
|
| 77 |
+
group: str,
|
| 78 |
+
stage_name: str,
|
| 79 |
+
overrides: Dict[Tuple[str, str], str],
|
| 80 |
+
) -> Tuple[str, bool]:
|
| 81 |
+
override = overrides.get((group, stage_name.lower()))
|
| 82 |
+
if override:
|
| 83 |
+
return override, os.path.isdir(override)
|
| 84 |
+
|
| 85 |
+
candidates = [
|
| 86 |
+
safe_str(stage_cfg.get("tokenizer_path")),
|
| 87 |
+
safe_str(stage_cfg.get("model_path")),
|
| 88 |
+
safe_str(stage_cfg.get("model_name")),
|
| 89 |
+
]
|
| 90 |
+
for candidate in candidates:
|
| 91 |
+
if candidate and os.path.isdir(candidate):
|
| 92 |
+
return candidate, True
|
| 93 |
+
for candidate in candidates:
|
| 94 |
+
if candidate and not os.path.isabs(candidate) and ("/" in candidate or Path(candidate).exists()):
|
| 95 |
+
return candidate, False
|
| 96 |
+
|
| 97 |
+
raise FileNotFoundError(
|
| 98 |
+
"No usable tokenizer source found for "
|
| 99 |
+
f"group={group} stage={stage_name}. "
|
| 100 |
+
f"Checked candidates={candidates}. "
|
| 101 |
+
"Pass --tokenizer-source <group>:<stage>=<source> to override."
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _load_tokenizer(
|
| 106 |
+
stage_cfg: Dict,
|
| 107 |
+
group: str,
|
| 108 |
+
stage_name: str,
|
| 109 |
+
overrides: Dict[Tuple[str, str], str],
|
| 110 |
+
):
|
| 111 |
+
source, local_files_only = _resolve_tokenizer_source(stage_cfg, group, stage_name, overrides)
|
| 112 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 113 |
+
source,
|
| 114 |
+
trust_remote_code=True,
|
| 115 |
+
local_files_only=local_files_only,
|
| 116 |
+
use_fast=False,
|
| 117 |
+
)
|
| 118 |
+
tokenizer.padding_side = "left"
|
| 119 |
+
if tokenizer.pad_token is None and tokenizer.eos_token is not None:
|
| 120 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 121 |
+
return tokenizer
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _compute_stage_cost(sample: Dict, stage_cfg: Dict, stage_index: int, tokenizer, retry: bool = False) -> float:
|
| 125 |
+
task_name = safe_str(sample.get("task"))
|
| 126 |
+
question = safe_str(sample.get("headline"))
|
| 127 |
+
rendered = render_chat(
|
| 128 |
+
tokenizer,
|
| 129 |
+
safe_str(stage_cfg.get("system_message")) or SYSTEM_MESSAGE_DEFAULT,
|
| 130 |
+
build_router_prompt_for_cost(question, task_name, stage_index, sample, retry=retry),
|
| 131 |
+
)
|
| 132 |
+
if retry:
|
| 133 |
+
raw_text = safe_str(sample.get(stage_retry_raw_key(stage_index))) or safe_str(sample.get(stage_retry_pred_key(stage_index)))
|
| 134 |
+
else:
|
| 135 |
+
raw_text = safe_str(sample.get(stage_raw_key(stage_index))) or safe_str(sample.get(stage_pred_key(stage_index)))
|
| 136 |
+
input_unit = _price_per_token(float(stage_cfg["input_cost_per_mtoken"]))
|
| 137 |
+
output_unit = _price_per_token(float(stage_cfg["output_cost_per_mtoken"]))
|
| 138 |
+
return count_tokens(tokenizer, rendered) * input_unit + count_tokens(tokenizer, raw_text) * output_unit
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _build_only_prompt(question: str, task_name: str) -> str:
|
| 142 |
+
return f"{task_instruction(task_name)}\n\nQuestion:\n{question}\n"
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _compute_only_cost(sample: Dict, stage_cfg: Dict, tokenizer) -> float:
|
| 146 |
+
task_name = safe_str(sample.get("task"))
|
| 147 |
+
question = safe_str(sample.get("headline"))
|
| 148 |
+
raw_text = safe_str(sample.get("raw")) or safe_str(sample.get("pred"))
|
| 149 |
+
rendered = render_chat(
|
| 150 |
+
tokenizer,
|
| 151 |
+
safe_str(stage_cfg.get("system_message")) or SYSTEM_MESSAGE_DEFAULT,
|
| 152 |
+
_build_only_prompt(question, task_name),
|
| 153 |
+
)
|
| 154 |
+
input_unit = _price_per_token(float(stage_cfg["input_cost_per_mtoken"]))
|
| 155 |
+
output_unit = _price_per_token(float(stage_cfg["output_cost_per_mtoken"]))
|
| 156 |
+
return count_tokens(tokenizer, rendered) * input_unit + count_tokens(tokenizer, raw_text) * output_unit
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _empty_stats(applicable: bool = True) -> Dict[str, Optional[float]]:
|
| 160 |
+
return {
|
| 161 |
+
"applicable": bool(applicable),
|
| 162 |
+
"samples": 0,
|
| 163 |
+
"correct": 0,
|
| 164 |
+
"accuracy": None,
|
| 165 |
+
"total_cost": 0.0,
|
| 166 |
+
"avg_cost": None,
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _finalize_stats(stats: Dict[str, Optional[float]]) -> Dict[str, Optional[float]]:
|
| 171 |
+
samples = int(stats["samples"])
|
| 172 |
+
if not samples:
|
| 173 |
+
stats["accuracy"] = None
|
| 174 |
+
stats["avg_cost"] = None
|
| 175 |
+
return stats
|
| 176 |
+
stats["accuracy"] = float(stats["correct"]) / samples
|
| 177 |
+
stats["avg_cost"] = float(stats["total_cost"]) / samples
|
| 178 |
+
return stats
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _stage_slot_name(stage_index: int) -> str:
|
| 182 |
+
return f"stage{stage_index + 1}"
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _init_stage_summary() -> Dict[str, Dict[str, Dict[str, Optional[float]]]]:
|
| 186 |
+
summary: Dict[str, Dict[str, Dict[str, Optional[float]]]] = {}
|
| 187 |
+
for stage_index in iter_stage_indices():
|
| 188 |
+
stage_name = _stage_slot_name(stage_index)
|
| 189 |
+
summary[stage_name] = {
|
| 190 |
+
"first": _empty_stats(applicable=True),
|
| 191 |
+
"rerun": _empty_stats(applicable=(stage_index < 2)),
|
| 192 |
+
}
|
| 193 |
+
return summary
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _merge_stats(dst: Dict[str, Optional[float]], src: Dict[str, Optional[float]]) -> None:
|
| 197 |
+
dst["samples"] = int(dst["samples"]) + int(src["samples"])
|
| 198 |
+
dst["correct"] = int(dst["correct"]) + int(src["correct"])
|
| 199 |
+
dst["total_cost"] = float(dst["total_cost"]) + float(src["total_cost"])
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _iter_cascade_targets(
|
| 203 |
+
root: Path,
|
| 204 |
+
benchmarks: Sequence[str],
|
| 205 |
+
groups: Sequence[str],
|
| 206 |
+
splits: Sequence[str],
|
| 207 |
+
) -> Iterable[Tuple[str, str, str, Path]]:
|
| 208 |
+
for benchmark in benchmarks:
|
| 209 |
+
for group in groups:
|
| 210 |
+
for split in splits:
|
| 211 |
+
path = root / benchmark / group / f"{split}.stage3.cached.jsonl"
|
| 212 |
+
if path.exists():
|
| 213 |
+
yield benchmark, group, split, path
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _iter_only_targets(
|
| 217 |
+
root: Path,
|
| 218 |
+
benchmarks: Sequence[str],
|
| 219 |
+
splits: Sequence[str],
|
| 220 |
+
) -> Iterable[Tuple[str, str, str, Path]]:
|
| 221 |
+
for benchmark in benchmarks:
|
| 222 |
+
only_dir = root / benchmark / "only"
|
| 223 |
+
if not only_dir.exists():
|
| 224 |
+
continue
|
| 225 |
+
for model_dir in sorted(path for path in only_dir.iterdir() if path.is_dir()):
|
| 226 |
+
for split in splits:
|
| 227 |
+
path = model_dir / f"{split}.jsonl"
|
| 228 |
+
if path.exists():
|
| 229 |
+
yield benchmark, model_dir.name, split, path
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _build_cascade_rows(
|
| 233 |
+
root: Path,
|
| 234 |
+
benchmarks: Sequence[str],
|
| 235 |
+
groups: Sequence[str],
|
| 236 |
+
splits: Sequence[str],
|
| 237 |
+
tokenizer_overrides: Dict[Tuple[str, str], str],
|
| 238 |
+
recompute_costs: bool,
|
| 239 |
+
) -> List[Dict]:
|
| 240 |
+
tokenizer_cache: Dict[Tuple[str, str], object] = {}
|
| 241 |
+
rows: List[Dict] = []
|
| 242 |
+
aggregate: Dict[Tuple[str, str], Dict] = {}
|
| 243 |
+
|
| 244 |
+
for benchmark, group, split, path in _iter_cascade_targets(root, benchmarks, groups, splits):
|
| 245 |
+
stage_cfgs = MODEL_GROUPS[group]["stages"]
|
| 246 |
+
samples = _read_jsonl(path)
|
| 247 |
+
if not samples:
|
| 248 |
+
continue
|
| 249 |
+
|
| 250 |
+
stage_summary = _init_stage_summary()
|
| 251 |
+
for sample in samples:
|
| 252 |
+
for stage_index in iter_stage_indices():
|
| 253 |
+
stage_name = _stage_slot_name(stage_index)
|
| 254 |
+
if stage_output_exists(sample, stage_index):
|
| 255 |
+
stats = stage_summary[stage_name]["first"]
|
| 256 |
+
stats["samples"] = int(stats["samples"]) + 1
|
| 257 |
+
stats["correct"] = int(stats["correct"]) + int(bool(sample.get(stage_correct_key(stage_index), False)))
|
| 258 |
+
cost_key = f"{stage_name}_cost"
|
| 259 |
+
if not recompute_costs and cost_key in sample:
|
| 260 |
+
cost_value = float(sample.get(cost_key, 0.0))
|
| 261 |
+
else:
|
| 262 |
+
tok_key = (group, stage_name)
|
| 263 |
+
if tok_key not in tokenizer_cache:
|
| 264 |
+
tokenizer_cache[tok_key] = _load_tokenizer(stage_cfgs[stage_index], group, stage_name, tokenizer_overrides)
|
| 265 |
+
cost_value = _compute_stage_cost(
|
| 266 |
+
sample, stage_cfgs[stage_index], stage_index, tokenizer_cache[tok_key], retry=False
|
| 267 |
+
)
|
| 268 |
+
stats["total_cost"] = float(stats["total_cost"]) + cost_value
|
| 269 |
+
|
| 270 |
+
if stage_index < 2 and stage_retry_output_exists(sample, stage_index):
|
| 271 |
+
stats = stage_summary[stage_name]["rerun"]
|
| 272 |
+
stats["samples"] = int(stats["samples"]) + 1
|
| 273 |
+
stats["correct"] = int(stats["correct"]) + int(bool(sample.get(stage_retry_correct_key(stage_index), False)))
|
| 274 |
+
retry_cost_key = f"{stage_name}_retry_cost"
|
| 275 |
+
if not recompute_costs and retry_cost_key in sample:
|
| 276 |
+
cost_value = float(sample.get(retry_cost_key, 0.0))
|
| 277 |
+
else:
|
| 278 |
+
tok_key = (group, stage_name)
|
| 279 |
+
if tok_key not in tokenizer_cache:
|
| 280 |
+
tokenizer_cache[tok_key] = _load_tokenizer(stage_cfgs[stage_index], group, stage_name, tokenizer_overrides)
|
| 281 |
+
cost_value = _compute_stage_cost(
|
| 282 |
+
sample, stage_cfgs[stage_index], stage_index, tokenizer_cache[tok_key], retry=True
|
| 283 |
+
)
|
| 284 |
+
stats["total_cost"] = float(stats["total_cost"]) + cost_value
|
| 285 |
+
|
| 286 |
+
for stage_name, slot_map in stage_summary.items():
|
| 287 |
+
for attempt_name, stats in slot_map.items():
|
| 288 |
+
_finalize_stats(stats)
|
| 289 |
+
|
| 290 |
+
row = {
|
| 291 |
+
"kind": "cascade",
|
| 292 |
+
"benchmark": benchmark,
|
| 293 |
+
"group": group,
|
| 294 |
+
"split": split,
|
| 295 |
+
"samples": len(samples),
|
| 296 |
+
"combo": safe_str(MODEL_GROUPS[group].get("display_name")) or group,
|
| 297 |
+
"stage_models": {
|
| 298 |
+
_stage_slot_name(idx): safe_str(stage_cfgs[idx].get("cache_id") or stage_cfgs[idx].get("model_name"))
|
| 299 |
+
for idx in iter_stage_indices()
|
| 300 |
+
},
|
| 301 |
+
"stages": stage_summary,
|
| 302 |
+
}
|
| 303 |
+
rows.append(row)
|
| 304 |
+
|
| 305 |
+
agg_key = (benchmark, group)
|
| 306 |
+
bucket = aggregate.setdefault(
|
| 307 |
+
agg_key,
|
| 308 |
+
{
|
| 309 |
+
"samples": 0,
|
| 310 |
+
"stage_models": row["stage_models"],
|
| 311 |
+
"combo": row["combo"],
|
| 312 |
+
"stages": _init_stage_summary(),
|
| 313 |
+
},
|
| 314 |
+
)
|
| 315 |
+
bucket["samples"] += len(samples)
|
| 316 |
+
for stage_name, slot_map in stage_summary.items():
|
| 317 |
+
for attempt_name, stats in slot_map.items():
|
| 318 |
+
_merge_stats(bucket["stages"][stage_name][attempt_name], stats)
|
| 319 |
+
|
| 320 |
+
final_rows: List[Dict] = []
|
| 321 |
+
for benchmark, group in sorted(aggregate):
|
| 322 |
+
bucket = aggregate[(benchmark, group)]
|
| 323 |
+
for stage_name, slot_map in bucket["stages"].items():
|
| 324 |
+
for attempt_name, stats in slot_map.items():
|
| 325 |
+
_finalize_stats(stats)
|
| 326 |
+
final_rows.append(
|
| 327 |
+
{
|
| 328 |
+
"kind": "cascade",
|
| 329 |
+
"benchmark": benchmark,
|
| 330 |
+
"group": group,
|
| 331 |
+
"split": "all",
|
| 332 |
+
"samples": bucket["samples"],
|
| 333 |
+
"combo": bucket["combo"],
|
| 334 |
+
"stage_models": bucket["stage_models"],
|
| 335 |
+
"stages": bucket["stages"],
|
| 336 |
+
}
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
rows.sort(key=lambda item: (item["benchmark"], item["group"], item["split"]))
|
| 340 |
+
final_rows.extend(rows)
|
| 341 |
+
return final_rows
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def _load_only_report(path: Path) -> Optional[Dict]:
|
| 345 |
+
report_path = path.with_suffix(".report.json")
|
| 346 |
+
if not report_path.exists():
|
| 347 |
+
return None
|
| 348 |
+
return json.loads(report_path.read_text(encoding="utf-8"))
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _resolve_only_model_specs() -> Dict[str, Tuple[str, str, Dict]]:
|
| 352 |
+
specs: Dict[str, Tuple[str, str, Dict]] = {}
|
| 353 |
+
for group, group_cfg in MODEL_GROUPS.items():
|
| 354 |
+
for stage_index, stage_cfg in enumerate(group_cfg["stages"]):
|
| 355 |
+
model_id = safe_str(stage_cfg.get("cache_id"))
|
| 356 |
+
if not model_id:
|
| 357 |
+
continue
|
| 358 |
+
stage_name = _stage_slot_name(stage_index)
|
| 359 |
+
model_key = model_id.lower()
|
| 360 |
+
existing = specs.get(model_key)
|
| 361 |
+
if existing is None:
|
| 362 |
+
specs[model_key] = (group, stage_name, dict(stage_cfg))
|
| 363 |
+
continue
|
| 364 |
+
existing_group, existing_stage_name, existing_cfg = existing
|
| 365 |
+
comparable_keys = (
|
| 366 |
+
"tokenizer_path",
|
| 367 |
+
"model_path",
|
| 368 |
+
"model_name",
|
| 369 |
+
"input_cost_per_mtoken",
|
| 370 |
+
"output_cost_per_mtoken",
|
| 371 |
+
"system_message",
|
| 372 |
+
)
|
| 373 |
+
if any(safe_str(existing_cfg.get(key)) != safe_str(stage_cfg.get(key)) for key in comparable_keys):
|
| 374 |
+
raise ValueError(
|
| 375 |
+
f"Conflicting only model config for {model_id}: "
|
| 376 |
+
f"{existing_group}/{existing_stage_name} vs {group}/{stage_name}"
|
| 377 |
+
)
|
| 378 |
+
return specs
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def _build_only_rows(
|
| 382 |
+
root: Path,
|
| 383 |
+
benchmarks: Sequence[str],
|
| 384 |
+
splits: Sequence[str],
|
| 385 |
+
tokenizer_overrides: Dict[Tuple[str, str], str],
|
| 386 |
+
) -> List[Dict]:
|
| 387 |
+
model_specs = _resolve_only_model_specs()
|
| 388 |
+
tokenizer_cache: Dict[Tuple[str, str], object] = {}
|
| 389 |
+
rows: List[Dict] = []
|
| 390 |
+
aggregate: Dict[Tuple[str, str], Dict[str, float]] = {}
|
| 391 |
+
|
| 392 |
+
for benchmark, model_id, split, path in _iter_only_targets(root, benchmarks, splits):
|
| 393 |
+
samples = _read_jsonl(path)
|
| 394 |
+
if not samples:
|
| 395 |
+
continue
|
| 396 |
+
|
| 397 |
+
model_key = model_id.lower()
|
| 398 |
+
if model_key not in model_specs:
|
| 399 |
+
raise KeyError(f"Missing current config for only model: {model_id}")
|
| 400 |
+
group, stage_name, stage_cfg = model_specs[model_key]
|
| 401 |
+
tok_key = (group, stage_name)
|
| 402 |
+
if tok_key not in tokenizer_cache:
|
| 403 |
+
tokenizer_cache[tok_key] = _load_tokenizer(stage_cfg, group, stage_name, tokenizer_overrides)
|
| 404 |
+
|
| 405 |
+
total = len(samples)
|
| 406 |
+
correct = 0
|
| 407 |
+
total_cost = 0.0
|
| 408 |
+
for sample in samples:
|
| 409 |
+
pred = safe_str(sample.get("pred"))
|
| 410 |
+
gold = sample.get("label")
|
| 411 |
+
task_name = safe_str(sample.get("task"))
|
| 412 |
+
correct += int(check_answer(task_name, pred, gold))
|
| 413 |
+
total_cost += _compute_only_cost(sample, stage_cfg, tokenizer_cache[tok_key])
|
| 414 |
+
|
| 415 |
+
accuracy = (correct / total) if total > 0 else None
|
| 416 |
+
avg_cost = (total_cost / total) if total > 0 else None
|
| 417 |
+
model_name = safe_str(samples[0].get("model_name")) or safe_str(stage_cfg.get("model_name")) or model_id
|
| 418 |
+
|
| 419 |
+
row = {
|
| 420 |
+
"kind": "only",
|
| 421 |
+
"benchmark": benchmark,
|
| 422 |
+
"group": "only",
|
| 423 |
+
"split": split,
|
| 424 |
+
"samples": total,
|
| 425 |
+
"model_id": model_id,
|
| 426 |
+
"model_name": model_name,
|
| 427 |
+
"correct": correct,
|
| 428 |
+
"accuracy": accuracy,
|
| 429 |
+
"total_cost": total_cost,
|
| 430 |
+
"avg_cost": avg_cost,
|
| 431 |
+
}
|
| 432 |
+
rows.append(row)
|
| 433 |
+
|
| 434 |
+
agg_key = (benchmark, model_id)
|
| 435 |
+
bucket = aggregate.setdefault(
|
| 436 |
+
agg_key,
|
| 437 |
+
{
|
| 438 |
+
"samples": 0,
|
| 439 |
+
"correct": 0,
|
| 440 |
+
"total_cost": 0.0,
|
| 441 |
+
"model_name": model_name,
|
| 442 |
+
},
|
| 443 |
+
)
|
| 444 |
+
bucket["samples"] += total
|
| 445 |
+
bucket["correct"] += correct
|
| 446 |
+
bucket["total_cost"] += total_cost
|
| 447 |
+
|
| 448 |
+
final_rows: List[Dict] = []
|
| 449 |
+
for benchmark, model_id in sorted(aggregate):
|
| 450 |
+
bucket = aggregate[(benchmark, model_id)]
|
| 451 |
+
samples = int(bucket["samples"])
|
| 452 |
+
accuracy = (int(bucket["correct"]) / samples) if samples > 0 else None
|
| 453 |
+
avg_cost = (float(bucket["total_cost"]) / samples) if samples > 0 else None
|
| 454 |
+
final_rows.append(
|
| 455 |
+
{
|
| 456 |
+
"kind": "only",
|
| 457 |
+
"benchmark": benchmark,
|
| 458 |
+
"group": "only",
|
| 459 |
+
"split": "all",
|
| 460 |
+
"samples": samples,
|
| 461 |
+
"model_id": model_id,
|
| 462 |
+
"model_name": bucket["model_name"],
|
| 463 |
+
"correct": int(bucket["correct"]),
|
| 464 |
+
"accuracy": accuracy,
|
| 465 |
+
"total_cost": float(bucket["total_cost"]),
|
| 466 |
+
"avg_cost": avg_cost,
|
| 467 |
+
}
|
| 468 |
+
)
|
| 469 |
+
|
| 470 |
+
rows.sort(key=lambda item: (item["benchmark"], item["model_id"], item["split"]))
|
| 471 |
+
final_rows.extend(rows)
|
| 472 |
+
return final_rows
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def _print_cascade_table(rows: Sequence[Dict]) -> None:
|
| 476 |
+
headers = [
|
| 477 |
+
"benchmark",
|
| 478 |
+
"group",
|
| 479 |
+
"split",
|
| 480 |
+
"samples",
|
| 481 |
+
"s1_first_acc",
|
| 482 |
+
"s1_rerun_acc",
|
| 483 |
+
"s2_first_acc",
|
| 484 |
+
"s2_rerun_acc",
|
| 485 |
+
"s3_first_acc",
|
| 486 |
+
"s3_rerun_acc",
|
| 487 |
+
"s1_first_cost",
|
| 488 |
+
"s1_rerun_cost",
|
| 489 |
+
"s2_first_cost",
|
| 490 |
+
"s2_rerun_cost",
|
| 491 |
+
"s3_first_cost",
|
| 492 |
+
"s3_rerun_cost",
|
| 493 |
+
]
|
| 494 |
+
str_rows: List[Dict[str, str]] = []
|
| 495 |
+
for row in rows:
|
| 496 |
+
if row.get("kind") != "cascade":
|
| 497 |
+
continue
|
| 498 |
+
stages = row["stages"]
|
| 499 |
+
str_rows.append(
|
| 500 |
+
{
|
| 501 |
+
"benchmark": str(row["benchmark"]),
|
| 502 |
+
"group": str(row["group"]),
|
| 503 |
+
"split": str(row["split"]),
|
| 504 |
+
"samples": str(row["samples"]),
|
| 505 |
+
"s1_first_acc": _fmt(stages["stage1"]["first"]["accuracy"]),
|
| 506 |
+
"s1_rerun_acc": _fmt(stages["stage1"]["rerun"]["accuracy"]),
|
| 507 |
+
"s2_first_acc": _fmt(stages["stage2"]["first"]["accuracy"]),
|
| 508 |
+
"s2_rerun_acc": _fmt(stages["stage2"]["rerun"]["accuracy"]),
|
| 509 |
+
"s3_first_acc": _fmt(stages["stage3"]["first"]["accuracy"]),
|
| 510 |
+
"s3_rerun_acc": _fmt(stages["stage3"]["rerun"]["accuracy"]),
|
| 511 |
+
"s1_first_cost": _fmt(stages["stage1"]["first"]["avg_cost"]),
|
| 512 |
+
"s1_rerun_cost": _fmt(stages["stage1"]["rerun"]["avg_cost"]),
|
| 513 |
+
"s2_first_cost": _fmt(stages["stage2"]["first"]["avg_cost"]),
|
| 514 |
+
"s2_rerun_cost": _fmt(stages["stage2"]["rerun"]["avg_cost"]),
|
| 515 |
+
"s3_first_cost": _fmt(stages["stage3"]["first"]["avg_cost"]),
|
| 516 |
+
"s3_rerun_cost": _fmt(stages["stage3"]["rerun"]["avg_cost"]),
|
| 517 |
+
}
|
| 518 |
+
)
|
| 519 |
+
if not str_rows:
|
| 520 |
+
return
|
| 521 |
+
widths = {
|
| 522 |
+
header: max(len(header), *(len(item[header]) for item in str_rows)) if str_rows else len(header)
|
| 523 |
+
for header in headers
|
| 524 |
+
}
|
| 525 |
+
print(" ".join(header.ljust(widths[header]) for header in headers))
|
| 526 |
+
print(" ".join("-" * widths[header] for header in headers))
|
| 527 |
+
for row in str_rows:
|
| 528 |
+
print(" ".join(row[header].ljust(widths[header]) for header in headers))
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
def _print_only_table(rows: Sequence[Dict]) -> None:
|
| 532 |
+
headers = ["benchmark", "model_id", "split", "samples", "accuracy", "avg_cost"]
|
| 533 |
+
str_rows: List[Dict[str, str]] = []
|
| 534 |
+
for row in rows:
|
| 535 |
+
if row.get("kind") != "only":
|
| 536 |
+
continue
|
| 537 |
+
str_rows.append(
|
| 538 |
+
{
|
| 539 |
+
"benchmark": str(row["benchmark"]),
|
| 540 |
+
"model_id": str(row["model_id"]),
|
| 541 |
+
"split": str(row["split"]),
|
| 542 |
+
"samples": str(row["samples"]),
|
| 543 |
+
"accuracy": _fmt(row["accuracy"]),
|
| 544 |
+
"avg_cost": _fmt(row["avg_cost"]),
|
| 545 |
+
}
|
| 546 |
+
)
|
| 547 |
+
if not str_rows:
|
| 548 |
+
return
|
| 549 |
+
print()
|
| 550 |
+
widths = {
|
| 551 |
+
header: max(len(header), *(len(item[header]) for item in str_rows)) if str_rows else len(header)
|
| 552 |
+
for header in headers
|
| 553 |
+
}
|
| 554 |
+
print(" ".join(header.ljust(widths[header]) for header in headers))
|
| 555 |
+
print(" ".join("-" * widths[header] for header in headers))
|
| 556 |
+
for row in str_rows:
|
| 557 |
+
print(" ".join(row[header].ljust(widths[header]) for header in headers))
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def _fmt(value: Optional[float]) -> str:
|
| 561 |
+
if value is None:
|
| 562 |
+
return "-"
|
| 563 |
+
return f"{float(value):.8f}"
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
def main() -> int:
|
| 567 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 568 |
+
parser.add_argument("--root", default="cache/benchmarks", help="Benchmark cache root directory.")
|
| 569 |
+
parser.add_argument("--benchmarks", nargs="*", help="Benchmarks to include. Default: all under cache root.")
|
| 570 |
+
parser.add_argument("--groups", nargs="*", choices=sorted(MODEL_GROUPS), help="Cascade model groups to include. Default: all.")
|
| 571 |
+
parser.add_argument("--splits", nargs="*", default=["train", "test"], help="Splits to include. Default: train test.")
|
| 572 |
+
parser.add_argument(
|
| 573 |
+
"--tokenizer-source",
|
| 574 |
+
action="append",
|
| 575 |
+
default=[],
|
| 576 |
+
help="Override tokenizer source. Format: <group>:<stage>=<source>, e.g. qwen:stage1=/path/to/tokenizer",
|
| 577 |
+
)
|
| 578 |
+
parser.add_argument(
|
| 579 |
+
"--recompute-costs",
|
| 580 |
+
action="store_true",
|
| 581 |
+
help="Ignore stored cost fields and recompute costs from tokenizer/pricing config.",
|
| 582 |
+
)
|
| 583 |
+
parser.add_argument("--output-json", help="Optional path to write JSON results.")
|
| 584 |
+
args = parser.parse_args()
|
| 585 |
+
|
| 586 |
+
root = Path(args.root)
|
| 587 |
+
if not root.exists():
|
| 588 |
+
raise FileNotFoundError(f"Benchmark cache root does not exist: {root}")
|
| 589 |
+
|
| 590 |
+
benchmarks = list(args.benchmarks) if args.benchmarks else sorted(
|
| 591 |
+
path.name for path in root.iterdir() if path.is_dir() and path.name != "only"
|
| 592 |
+
)
|
| 593 |
+
groups = list(args.groups) if args.groups else sorted(MODEL_GROUPS)
|
| 594 |
+
splits = [str(split).strip() for split in args.splits if str(split).strip()]
|
| 595 |
+
tokenizer_overrides = _parse_tokenizer_overrides(args.tokenizer_source)
|
| 596 |
+
|
| 597 |
+
cascade_rows = _build_cascade_rows(root, benchmarks, groups, splits, tokenizer_overrides, args.recompute_costs)
|
| 598 |
+
only_rows = _build_only_rows(root, benchmarks, splits, tokenizer_overrides)
|
| 599 |
+
final_rows = cascade_rows + only_rows
|
| 600 |
+
|
| 601 |
+
_print_cascade_table(cascade_rows)
|
| 602 |
+
_print_only_table(only_rows)
|
| 603 |
+
|
| 604 |
+
if args.output_json:
|
| 605 |
+
output_path = Path(args.output_json)
|
| 606 |
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 607 |
+
payload = {
|
| 608 |
+
"root": str(root),
|
| 609 |
+
"benchmarks": benchmarks,
|
| 610 |
+
"groups": groups,
|
| 611 |
+
"splits": splits,
|
| 612 |
+
"cascade_results": cascade_rows,
|
| 613 |
+
"only_results": only_rows,
|
| 614 |
+
"results": final_rows,
|
| 615 |
+
}
|
| 616 |
+
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 617 |
+
|
| 618 |
+
return 0
|
| 619 |
+
|
| 620 |
+
|
| 621 |
+
if __name__ == "__main__":
|
| 622 |
+
raise SystemExit(main())
|
report_stage_recovery.py
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Report conditional correction rates for cascade upgrades versus only models."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
| 11 |
+
|
| 12 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 13 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 14 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 15 |
+
|
| 16 |
+
from answer_utils import check_answer
|
| 17 |
+
from config import MODEL_GROUPS
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _safe_str(value: Any) -> str:
|
| 21 |
+
if value is None:
|
| 22 |
+
return ""
|
| 23 |
+
return str(value).strip()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
|
| 27 |
+
rows: List[Dict[str, Any]] = []
|
| 28 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 29 |
+
for line in handle:
|
| 30 |
+
line = line.strip()
|
| 31 |
+
if line:
|
| 32 |
+
rows.append(json.loads(line))
|
| 33 |
+
return rows
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _cache_key(row: Dict[str, Any], fallback_index: int) -> int:
|
| 37 |
+
raw_key = row.get("_cache_index")
|
| 38 |
+
if raw_key is None:
|
| 39 |
+
return int(fallback_index)
|
| 40 |
+
return int(raw_key)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _bool_correct(task: str, pred: Any, gold: Any) -> bool:
|
| 44 |
+
return bool(check_answer(task, _safe_str(pred), gold))
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _stage_pred(row: Dict[str, Any], stage_index: int) -> str:
|
| 48 |
+
return _safe_str(row.get(f"stage{stage_index + 1}_pred"))
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _stage_correct(row: Dict[str, Any], stage_index: int) -> bool:
|
| 52 |
+
return _bool_correct(_safe_str(row.get("task")), _stage_pred(row, stage_index), row.get("label"))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _only_pred(row: Dict[str, Any]) -> str:
|
| 56 |
+
return _safe_str(row.get("pred"))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _only_correct(row: Dict[str, Any]) -> bool:
|
| 60 |
+
return _bool_correct(_safe_str(row.get("task")), _only_pred(row), row.get("label"))
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _iter_benchmarks(root: Path, requested: Optional[Sequence[str]]) -> List[str]:
|
| 64 |
+
if requested:
|
| 65 |
+
return [str(name).strip() for name in requested if str(name).strip()]
|
| 66 |
+
return sorted(path.name for path in root.iterdir() if path.is_dir())
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _load_stage_rows(root: Path, benchmark: str, group: str, split: str) -> List[Dict[str, Any]]:
|
| 70 |
+
path = root / benchmark / group / f"{split}.stage3.cached.jsonl"
|
| 71 |
+
if not path.exists():
|
| 72 |
+
raise FileNotFoundError(path)
|
| 73 |
+
return _read_jsonl(path)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _load_only_rows(root: Path, benchmark: str, model_id: str, split: str) -> Dict[int, Dict[str, Any]]:
|
| 77 |
+
path = root / benchmark / "only" / model_id.lower() / f"{split}.jsonl"
|
| 78 |
+
if not path.exists():
|
| 79 |
+
raise FileNotFoundError(path)
|
| 80 |
+
rows = _read_jsonl(path)
|
| 81 |
+
indexed: Dict[int, Dict[str, Any]] = {}
|
| 82 |
+
for fallback_index, row in enumerate(rows):
|
| 83 |
+
indexed[_cache_key(row, fallback_index)] = row
|
| 84 |
+
return indexed
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _make_bucket() -> Dict[str, Any]:
|
| 88 |
+
return {
|
| 89 |
+
"denominator": 0,
|
| 90 |
+
"cascade_correct": 0,
|
| 91 |
+
"only_correct": 0,
|
| 92 |
+
"both_correct": 0,
|
| 93 |
+
"cascade_only_correct": 0,
|
| 94 |
+
"only_only_correct": 0,
|
| 95 |
+
"both_wrong": 0,
|
| 96 |
+
"missing_only": 0,
|
| 97 |
+
"mismatched_examples": 0,
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _finalize_bucket(
|
| 102 |
+
benchmark: str,
|
| 103 |
+
group: str,
|
| 104 |
+
split: str,
|
| 105 |
+
prev_stage_index: int,
|
| 106 |
+
next_stage_index: int,
|
| 107 |
+
prev_model_id: str,
|
| 108 |
+
next_model_id: str,
|
| 109 |
+
only_model_id: str,
|
| 110 |
+
bucket: Dict[str, Any],
|
| 111 |
+
) -> Dict[str, Any]:
|
| 112 |
+
denom = int(bucket["denominator"])
|
| 113 |
+
cascade_rate = (int(bucket["cascade_correct"]) / denom) if denom else None
|
| 114 |
+
only_rate = (int(bucket["only_correct"]) / denom) if denom else None
|
| 115 |
+
delta = (cascade_rate - only_rate) if (cascade_rate is not None and only_rate is not None) else None
|
| 116 |
+
return {
|
| 117 |
+
"benchmark": benchmark,
|
| 118 |
+
"group": group,
|
| 119 |
+
"split": split,
|
| 120 |
+
"transition": f"stage{prev_stage_index + 1}_wrong_to_stage{next_stage_index + 1}_correct",
|
| 121 |
+
"prev_stage": f"stage{prev_stage_index + 1}",
|
| 122 |
+
"next_stage": f"stage{next_stage_index + 1}",
|
| 123 |
+
"prev_model_id": prev_model_id,
|
| 124 |
+
"next_model_id": next_model_id,
|
| 125 |
+
"only_model_id": only_model_id,
|
| 126 |
+
"denominator": denom,
|
| 127 |
+
"cascade_correct": int(bucket["cascade_correct"]),
|
| 128 |
+
"cascade_rate": cascade_rate,
|
| 129 |
+
"only_correct": int(bucket["only_correct"]),
|
| 130 |
+
"only_rate": only_rate,
|
| 131 |
+
"delta_cascade_minus_only": delta,
|
| 132 |
+
"both_correct": int(bucket["both_correct"]),
|
| 133 |
+
"cascade_only_correct": int(bucket["cascade_only_correct"]),
|
| 134 |
+
"only_only_correct": int(bucket["only_only_correct"]),
|
| 135 |
+
"both_wrong": int(bucket["both_wrong"]),
|
| 136 |
+
"missing_only": int(bucket["missing_only"]),
|
| 137 |
+
"mismatched_examples": int(bucket["mismatched_examples"]),
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _analyze_transition(
|
| 142 |
+
benchmark: str,
|
| 143 |
+
group: str,
|
| 144 |
+
split: str,
|
| 145 |
+
stage_rows: List[Dict[str, Any]],
|
| 146 |
+
only_rows: Dict[int, Dict[str, Any]],
|
| 147 |
+
prev_stage_index: int,
|
| 148 |
+
next_stage_index: int,
|
| 149 |
+
prev_model_id: str,
|
| 150 |
+
next_model_id: str,
|
| 151 |
+
only_model_id: str,
|
| 152 |
+
) -> Dict[str, Any]:
|
| 153 |
+
bucket = _make_bucket()
|
| 154 |
+
for fallback_index, row in enumerate(stage_rows):
|
| 155 |
+
if _stage_correct(row, prev_stage_index):
|
| 156 |
+
continue
|
| 157 |
+
bucket["denominator"] += 1
|
| 158 |
+
cache_index = _cache_key(row, fallback_index)
|
| 159 |
+
only_row = only_rows.get(cache_index)
|
| 160 |
+
if only_row is None:
|
| 161 |
+
bucket["missing_only"] += 1
|
| 162 |
+
continue
|
| 163 |
+
|
| 164 |
+
if (
|
| 165 |
+
_safe_str(only_row.get("headline")) != _safe_str(row.get("headline"))
|
| 166 |
+
or _safe_str(only_row.get("label")) != _safe_str(row.get("label"))
|
| 167 |
+
):
|
| 168 |
+
bucket["mismatched_examples"] += 1
|
| 169 |
+
|
| 170 |
+
cascade_ok = _stage_correct(row, next_stage_index)
|
| 171 |
+
only_ok = _only_correct(only_row)
|
| 172 |
+
bucket["cascade_correct"] += int(cascade_ok)
|
| 173 |
+
bucket["only_correct"] += int(only_ok)
|
| 174 |
+
if cascade_ok and only_ok:
|
| 175 |
+
bucket["both_correct"] += 1
|
| 176 |
+
elif cascade_ok:
|
| 177 |
+
bucket["cascade_only_correct"] += 1
|
| 178 |
+
elif only_ok:
|
| 179 |
+
bucket["only_only_correct"] += 1
|
| 180 |
+
else:
|
| 181 |
+
bucket["both_wrong"] += 1
|
| 182 |
+
|
| 183 |
+
return _finalize_bucket(
|
| 184 |
+
benchmark=benchmark,
|
| 185 |
+
group=group,
|
| 186 |
+
split=split,
|
| 187 |
+
prev_stage_index=prev_stage_index,
|
| 188 |
+
next_stage_index=next_stage_index,
|
| 189 |
+
prev_model_id=prev_model_id,
|
| 190 |
+
next_model_id=next_model_id,
|
| 191 |
+
only_model_id=only_model_id,
|
| 192 |
+
bucket=bucket,
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _iter_rows(
|
| 197 |
+
root: Path,
|
| 198 |
+
benchmarks: Sequence[str],
|
| 199 |
+
groups: Sequence[str],
|
| 200 |
+
splits: Sequence[str],
|
| 201 |
+
) -> Iterable[Dict[str, Any]]:
|
| 202 |
+
for benchmark in benchmarks:
|
| 203 |
+
for group in groups:
|
| 204 |
+
stage_cfgs = MODEL_GROUPS[group]["stages"]
|
| 205 |
+
for split in splits:
|
| 206 |
+
try:
|
| 207 |
+
stage_rows = _load_stage_rows(root, benchmark, group, split)
|
| 208 |
+
except FileNotFoundError:
|
| 209 |
+
continue
|
| 210 |
+
for prev_stage_index, next_stage_index in ((0, 1), (1, 2)):
|
| 211 |
+
only_model_id = _safe_str(stage_cfgs[next_stage_index].get("cache_id"))
|
| 212 |
+
if not only_model_id:
|
| 213 |
+
continue
|
| 214 |
+
try:
|
| 215 |
+
only_rows = _load_only_rows(root, benchmark, only_model_id, split)
|
| 216 |
+
except FileNotFoundError:
|
| 217 |
+
continue
|
| 218 |
+
yield _analyze_transition(
|
| 219 |
+
benchmark=benchmark,
|
| 220 |
+
group=group,
|
| 221 |
+
split=split,
|
| 222 |
+
stage_rows=stage_rows,
|
| 223 |
+
only_rows=only_rows,
|
| 224 |
+
prev_stage_index=prev_stage_index,
|
| 225 |
+
next_stage_index=next_stage_index,
|
| 226 |
+
prev_model_id=_safe_str(stage_cfgs[prev_stage_index].get("cache_id")),
|
| 227 |
+
next_model_id=_safe_str(stage_cfgs[next_stage_index].get("cache_id")),
|
| 228 |
+
only_model_id=only_model_id,
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _fmt_rate(value: Optional[float]) -> str:
|
| 233 |
+
if value is None:
|
| 234 |
+
return "-"
|
| 235 |
+
return f"{100.0 * float(value):.2f}%"
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _fmt_count_rate(count: int, denom: int) -> str:
|
| 239 |
+
if not denom:
|
| 240 |
+
return "0/0"
|
| 241 |
+
return f"{int(count)}/{int(denom)} ({_fmt_rate(count / denom)})"
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _print_rows(rows: Sequence[Dict[str, Any]]) -> None:
|
| 245 |
+
headers = [
|
| 246 |
+
"benchmark",
|
| 247 |
+
"group",
|
| 248 |
+
"split",
|
| 249 |
+
"transition",
|
| 250 |
+
"subset",
|
| 251 |
+
"cascade",
|
| 252 |
+
"only",
|
| 253 |
+
"delta",
|
| 254 |
+
"overlap",
|
| 255 |
+
]
|
| 256 |
+
printable: List[Dict[str, str]] = []
|
| 257 |
+
for row in rows:
|
| 258 |
+
denom = int(row["denominator"])
|
| 259 |
+
printable.append(
|
| 260 |
+
{
|
| 261 |
+
"benchmark": str(row["benchmark"]),
|
| 262 |
+
"group": str(row["group"]),
|
| 263 |
+
"split": str(row["split"]),
|
| 264 |
+
"transition": f"{row['prev_model_id']} wrong -> {row['next_model_id']} correct",
|
| 265 |
+
"subset": str(denom),
|
| 266 |
+
"cascade": _fmt_count_rate(int(row["cascade_correct"]), denom),
|
| 267 |
+
"only": _fmt_count_rate(int(row["only_correct"]), denom),
|
| 268 |
+
"delta": _fmt_rate(row["delta_cascade_minus_only"]),
|
| 269 |
+
"overlap": (
|
| 270 |
+
f"both={row['both_correct']} "
|
| 271 |
+
f"c_only={row['cascade_only_correct']} "
|
| 272 |
+
f"o_only={row['only_only_correct']} "
|
| 273 |
+
f"none={row['both_wrong']}"
|
| 274 |
+
),
|
| 275 |
+
}
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
if not printable:
|
| 279 |
+
return
|
| 280 |
+
|
| 281 |
+
widths = {
|
| 282 |
+
header: max(len(header), *(len(item[header]) for item in printable))
|
| 283 |
+
for header in headers
|
| 284 |
+
}
|
| 285 |
+
print(" ".join(header.ljust(widths[header]) for header in headers))
|
| 286 |
+
print(" ".join(("-" * widths[header]) for header in headers))
|
| 287 |
+
for row in printable:
|
| 288 |
+
print(" ".join(row[header].ljust(widths[header]) for header in headers))
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def main() -> int:
|
| 292 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 293 |
+
parser.add_argument("--root", default="cache/benchmarks", help="Benchmark cache root.")
|
| 294 |
+
parser.add_argument("--benchmarks", nargs="*", help="Benchmarks to include. Default: all.")
|
| 295 |
+
parser.add_argument("--groups", nargs="*", choices=sorted(MODEL_GROUPS), help="Cascade groups to include. Default: all.")
|
| 296 |
+
parser.add_argument("--splits", nargs="*", default=["test"], help="Splits to include. Default: test.")
|
| 297 |
+
parser.add_argument("--output-json", help="Optional path to write JSON results.")
|
| 298 |
+
args = parser.parse_args()
|
| 299 |
+
|
| 300 |
+
root = Path(args.root)
|
| 301 |
+
if not root.exists():
|
| 302 |
+
raise FileNotFoundError(f"Benchmark cache root does not exist: {root}")
|
| 303 |
+
|
| 304 |
+
benchmarks = _iter_benchmarks(root, args.benchmarks)
|
| 305 |
+
groups = list(args.groups) if args.groups else sorted(MODEL_GROUPS)
|
| 306 |
+
splits = [str(split).strip() for split in args.splits if str(split).strip()]
|
| 307 |
+
rows = list(_iter_rows(root, benchmarks, groups, splits))
|
| 308 |
+
rows.sort(key=lambda item: (item["benchmark"], item["group"], item["split"], item["prev_stage"], item["next_stage"]))
|
| 309 |
+
|
| 310 |
+
_print_rows(rows)
|
| 311 |
+
|
| 312 |
+
if args.output_json:
|
| 313 |
+
output_path = Path(args.output_json)
|
| 314 |
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 315 |
+
payload = {
|
| 316 |
+
"root": str(root),
|
| 317 |
+
"benchmarks": benchmarks,
|
| 318 |
+
"groups": groups,
|
| 319 |
+
"splits": splits,
|
| 320 |
+
"rows": rows,
|
| 321 |
+
}
|
| 322 |
+
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 323 |
+
return 0
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
if __name__ == "__main__":
|
| 327 |
+
raise SystemExit(main())
|
tensorrt.sh
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
TENSORRT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/TensorRT-10.13.0.35
|
| 2 |
+
|
| 3 |
+
grep -q 'export TensorRT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/TensorRT-10.13.0.35' ~/.bashrc || cat >> ~/.bashrc <<EOF
|
| 4 |
+
|
| 5 |
+
# TensorRT
|
| 6 |
+
export TensorRT_ROOT=$TENSORRT_ROOT
|
| 7 |
+
export LD_LIBRARY_PATH=\$TensorRT_ROOT/lib:\${LD_LIBRARY_PATH}
|
| 8 |
+
EOF
|
| 9 |
+
|
| 10 |
+
source ~/.bashrc
|
| 11 |
+
|
| 12 |
+
# 检查
|
| 13 |
+
echo $TensorRT_ROOT
|
| 14 |
+
echo $LD_LIBRARY_PATH
|