Tinghong-Ye22 commited on
Commit
734d13f
·
verified ·
1 Parent(s): 82a0dc9

Add files using upload-large-folder tool

Browse files
backfill_stage_cache_costs.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Backfill or recompute cost fields for all staged benchmark cache jsonl files."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import Dict, Iterable, List
11
+
12
+ PROJECT_ROOT = Path(__file__).resolve().parents[1]
13
+ if str(PROJECT_ROOT) not in sys.path:
14
+ sys.path.insert(0, str(PROJECT_ROOT))
15
+
16
+ from answer_utils import repair_confidence_reason_record_fields
17
+ from cascade_utils import attach_dynamic_costs
18
+
19
+
20
+ def _read_jsonl(path: Path) -> List[Dict]:
21
+ rows: List[Dict] = []
22
+ with path.open("r", encoding="utf-8") as handle:
23
+ for line in handle:
24
+ line = line.strip()
25
+ if line:
26
+ rows.append(repair_confidence_reason_record_fields(json.loads(line)))
27
+ return rows
28
+
29
+
30
+ def _write_jsonl(path: Path, rows: List[Dict]) -> None:
31
+ path.parent.mkdir(parents=True, exist_ok=True)
32
+ with path.open("w", encoding="utf-8") as handle:
33
+ for row in rows:
34
+ handle.write(json.dumps(row, ensure_ascii=False) + "\n")
35
+
36
+
37
+ def _iter_cache_files(root: Path) -> Iterable[Path]:
38
+ for path in sorted(root.rglob("*.stage*.cached.jsonl")):
39
+ if path.is_file():
40
+ yield path
41
+
42
+
43
+ def main() -> int:
44
+ parser = argparse.ArgumentParser(description=__doc__)
45
+ parser.add_argument("--root", default="cache/benchmarks", help="Stage cache root directory.")
46
+ parser.add_argument(
47
+ "--missing-only",
48
+ action="store_true",
49
+ help="Only fill missing cost fields. Default behavior is to recompute all cost fields with current pricing.",
50
+ )
51
+ args = parser.parse_args()
52
+
53
+ root = Path(args.root)
54
+ if not root.exists():
55
+ raise FileNotFoundError(f"Cache root does not exist: {root}")
56
+
57
+ force_recompute = not args.missing_only
58
+ total_files = 0
59
+ total_rows = 0
60
+
61
+ for path in _iter_cache_files(root):
62
+ rows = _read_jsonl(path)
63
+ if not rows:
64
+ print(f"[Skip] empty {path}")
65
+ continue
66
+ changed = attach_dynamic_costs(rows, force_recompute=force_recompute)
67
+ if changed:
68
+ _write_jsonl(path, rows)
69
+ print(f"[Done] {path} rows={len(rows)} recompute={'yes' if force_recompute else 'missing-only'}")
70
+ else:
71
+ print(f"[Skip] up-to-date {path} rows={len(rows)}")
72
+ total_files += 1
73
+ total_rows += len(rows)
74
+
75
+ print(f"[Summary] files={total_files} rows={total_rows} mode={'recompute' if force_recompute else 'missing-only'}")
76
+ return 0
77
+
78
+
79
+ if __name__ == "__main__":
80
+ raise SystemExit(main())
drop_net_outputs.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
3
+ from pathlib import Path
4
+ import argparse
5
+ import os
6
+ import torch
7
+ from tqdm import tqdm
8
+
9
+
10
+ def load_pt(path: Path):
11
+ try:
12
+ return torch.load(path, map_location="cpu", weights_only=False)
13
+ except TypeError:
14
+ return torch.load(path, map_location="cpu")
15
+
16
+
17
+ def iter_pt_files(root: Path):
18
+ for p in root.rglob("*.pt"):
19
+ if p.is_file():
20
+ yield p
21
+
22
+
23
+ def process_one(path: Path, key: str):
24
+ try:
25
+ obj = load_pt(path)
26
+ if isinstance(obj, dict) and key in obj:
27
+ obj.pop(key, None)
28
+ torch.save(obj, path)
29
+ return 1, 0
30
+ return 0, 0
31
+ except Exception:
32
+ return 0, 1
33
+
34
+
35
+ def main():
36
+ parser = argparse.ArgumentParser(description="Drop a key from all .pt files recursively.")
37
+ parser.add_argument("root", nargs="?", default="/cpfs01/videogen/gvhmr_smpl")
38
+ parser.add_argument("--key", default="net_outputs")
39
+ parser.add_argument("--workers", type=int, default=min(16, os.cpu_count() or 1))
40
+ parser.add_argument("--prefetch", type=int, default=4)
41
+ args = parser.parse_args()
42
+
43
+ root = Path(args.root)
44
+ files = iter_pt_files(root)
45
+ workers = max(1, args.workers)
46
+ inflight_cap = max(1, workers * max(1, args.prefetch))
47
+ scanned = 0
48
+ removed = 0
49
+ errors = 0
50
+
51
+ print(f"start: root={root}, workers={workers}", flush=True)
52
+ with ThreadPoolExecutor(max_workers=workers) as pool, tqdm(
53
+ desc="processing", unit="file", dynamic_ncols=True
54
+ ) as pbar:
55
+ inflight = set()
56
+ for _ in range(inflight_cap):
57
+ try:
58
+ f = next(files)
59
+ except StopIteration:
60
+ break
61
+ inflight.add(pool.submit(process_one, f, args.key))
62
+
63
+ while inflight:
64
+ done, inflight = wait(inflight, return_when=FIRST_COMPLETED)
65
+ for fut in done:
66
+ r, e = fut.result()
67
+ scanned += 1
68
+ removed += r
69
+ errors += e
70
+ pbar.update(len(done))
71
+
72
+ for _ in range(len(done)):
73
+ try:
74
+ f = next(files)
75
+ except StopIteration:
76
+ break
77
+ inflight.add(pool.submit(process_one, f, args.key))
78
+
79
+ print(f"done: {scanned} files scanned, {removed} files updated, {errors} errors")
80
+
81
+
82
+ if __name__ == "__main__":
83
+ main()
extract_gvhmr_tars.sh ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ SRC_DIR="${SRC_DIR:-/cpfs01/videogen/gvhmr_res}"
5
+ DST_DIR="${DST_DIR:-/cpfs01/videogen/gvhmr_smpl}"
6
+
7
+ cpu_count="$(nproc 2>/dev/null || echo 8)"
8
+ default_jobs="$cpu_count"
9
+ if (( default_jobs > 8 )); then
10
+ default_jobs=8
11
+ fi
12
+ JOBS="${JOBS:-$default_jobs}"
13
+
14
+ usage() {
15
+ cat <<EOF
16
+ Usage: $(basename "$0") [-j JOBS] [-s SRC_DIR] [-d DST_DIR]
17
+
18
+ Options:
19
+ -j JOBS 并行解压进程数 (默认: min(nproc, 8))
20
+ -s SRC_DIR 压缩包目录 (默认: /cpfs01/videogen/gvhmr_res)
21
+ -d DST_DIR 解压目标目录 (默认: /cpfs01/videogen/gvhmr_smpl)
22
+ -h 显示帮助
23
+ EOF
24
+ }
25
+
26
+ while getopts ":j:s:d:h" opt; do
27
+ case "$opt" in
28
+ j) JOBS="$OPTARG" ;;
29
+ s) SRC_DIR="$OPTARG" ;;
30
+ d) DST_DIR="$OPTARG" ;;
31
+ h)
32
+ usage
33
+ exit 0
34
+ ;;
35
+ :)
36
+ echo "Error: option -$OPTARG requires an argument." >&2
37
+ usage
38
+ exit 1
39
+ ;;
40
+ \?)
41
+ echo "Error: invalid option -$OPTARG" >&2
42
+ usage
43
+ exit 1
44
+ ;;
45
+ esac
46
+ done
47
+
48
+ if ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then
49
+ echo "Error: JOBS must be a positive integer, got: $JOBS" >&2
50
+ exit 1
51
+ fi
52
+
53
+ if [[ ! -d "$SRC_DIR" ]]; then
54
+ echo "Error: source directory not found: $SRC_DIR" >&2
55
+ exit 1
56
+ fi
57
+
58
+ mkdir -p "$DST_DIR"
59
+
60
+ strip_archive_ext() {
61
+ local name="$1"
62
+ case "$name" in
63
+ *.tar.gz) printf '%s' "${name%.tar.gz}" ;;
64
+ *.tgz) printf '%s' "${name%.tgz}" ;;
65
+ *.tar.xz) printf '%s' "${name%.tar.xz}" ;;
66
+ *.txz) printf '%s' "${name%.txz}" ;;
67
+ *.tar.bz2) printf '%s' "${name%.tar.bz2}" ;;
68
+ *.tbz2) printf '%s' "${name%.tbz2}" ;;
69
+ *.tar.zst) printf '%s' "${name%.tar.zst}" ;;
70
+ *.tzst) printf '%s' "${name%.tzst}" ;;
71
+ *.tar) printf '%s' "${name%.tar}" ;;
72
+ *) printf '%s' "$name" ;;
73
+ esac
74
+ }
75
+
76
+ render_progress() {
77
+ local done="$1"
78
+ local total="$2"
79
+ local width=40
80
+ local filled=$(( done * width / total ))
81
+ local empty=$(( width - filled ))
82
+ local percent=$(( done * 100 / total ))
83
+ local bar_fill bar_empty
84
+
85
+ printf -v bar_fill "%${filled}s" ""
86
+ bar_fill="${bar_fill// /#}"
87
+ printf -v bar_empty "%${empty}s" ""
88
+ bar_empty="${bar_empty// /-}"
89
+
90
+ printf "\r[%s%s] %d/%d (%3d%%)" "$bar_fill" "$bar_empty" "$done" "$total" "$percent"
91
+ }
92
+
93
+ extract_one() {
94
+ local tar_file="$1"
95
+ local rel stem out_dir
96
+
97
+ rel="${tar_file#"$SRC_DIR"/}"
98
+ if [[ "$rel" == "$tar_file" ]]; then
99
+ rel="$(basename "$tar_file")"
100
+ fi
101
+
102
+ stem="$(strip_archive_ext "$rel")"
103
+ out_dir="$DST_DIR/$stem"
104
+
105
+ mkdir -p "$out_dir"
106
+ tar -xf "$tar_file" -C "$out_dir"
107
+ printf '%s\n' "$tar_file"
108
+ }
109
+
110
+ export SRC_DIR DST_DIR
111
+ export -f strip_archive_ext extract_one
112
+
113
+ mapfile -d '' TAR_FILES < <(
114
+ find "$SRC_DIR" -type f \
115
+ \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.xz' -o -name '*.txz' -o -name '*.tar.bz2' -o -name '*.tbz2' -o -name '*.tar.zst' -o -name '*.tzst' \) \
116
+ -print0 | sort -z
117
+ )
118
+
119
+ TOTAL="${#TAR_FILES[@]}"
120
+ if (( TOTAL == 0 )); then
121
+ echo "No tar archives found in: $SRC_DIR"
122
+ exit 0
123
+ fi
124
+
125
+ echo "Source: $SRC_DIR"
126
+ echo "Target: $DST_DIR"
127
+ echo "Archives: $TOTAL"
128
+ echo "Jobs: $JOBS"
129
+
130
+ start_ts="$(date +%s)"
131
+ done_count=0
132
+ render_progress 0 "$TOTAL"
133
+
134
+ printf '%s\0' "${TAR_FILES[@]}" \
135
+ | xargs -0 -n1 -P "$JOBS" bash -c 'extract_one "$1"' _ \
136
+ | while IFS= read -r _; do
137
+ done_count=$((done_count + 1))
138
+ render_progress "$done_count" "$TOTAL"
139
+ done
140
+
141
+ echo
142
+ end_ts="$(date +%s)"
143
+ elapsed="$((end_ts - start_ts))"
144
+ echo "Done in ${elapsed}s."
fast_rm.sh ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ usage() {
5
+ echo "Usage: bash $0 TARGET_DIR [JOBS]" >&2
6
+ exit 2
7
+ }
8
+
9
+ die() {
10
+ echo "fast_rm: $*" >&2
11
+ exit 2
12
+ }
13
+
14
+ format_bytes() {
15
+ awk -v bytes="${1:-0}" 'BEGIN {
16
+ split("B KiB MiB GiB TiB PiB", units, " ")
17
+ value = bytes + 0
18
+ unit = 1
19
+ while (value >= 1024 && unit < 6) {
20
+ value /= 1024
21
+ unit++
22
+ }
23
+ if (unit == 1) {
24
+ printf "%.0f %s", value, units[unit]
25
+ } else {
26
+ printf "%.2f %s", value, units[unit]
27
+ }
28
+ }'
29
+ }
30
+
31
+ fs_avail_bytes() {
32
+ df -B1 --output=avail "$1" 2>/dev/null | awk 'NR==2 {print $1}'
33
+ }
34
+
35
+ progress_bar() {
36
+ local freed=$1
37
+ local total=$2
38
+ local items=$3
39
+ local final=${4:-0}
40
+ local now elapsed percent filled empty bar rate capped
41
+ local width=30
42
+
43
+ now=$(date +%s)
44
+ elapsed=$((now - start_ts))
45
+ [ "$elapsed" -gt 0 ] || elapsed=1
46
+ [ "$freed" -ge 0 ] || freed=0
47
+ rate=$(awk -v b="$freed" -v s="$elapsed" 'BEGIN { if (s <= 0) s = 1; printf "%.0f", b / s }')
48
+
49
+ if [ "$total" -gt 0 ]; then
50
+ capped=$freed
51
+ [ "$capped" -le "$total" ] || capped=$total
52
+ percent=$((capped * 100 / total))
53
+ [ "$percent" -le 100 ] || percent=100
54
+ else
55
+ percent=0
56
+ fi
57
+
58
+ if [ "$total" -gt 0 ]; then
59
+ filled=$((percent * width / 100))
60
+ empty=$((width - filled))
61
+ bar=$(printf '%*s' "$filled" '' | tr ' ' '#')
62
+ bar=$bar$(printf '%*s' "$empty" '' | tr ' ' '-')
63
+ printf '\rfast_rm: [%s] %3d%% freed %s/%s rate %s/s items %d' \
64
+ "$bar" "$percent" "$(format_bytes "$freed")" "$(format_bytes "$total")" "$(format_bytes "$rate")" "$items" >&2
65
+ else
66
+ printf '\rfast_rm: freed %s rate %s/s items %d' \
67
+ "$(format_bytes "$freed")" "$(format_bytes "$rate")" "$items" >&2
68
+ fi
69
+
70
+ if [ "$final" = 1 ]; then
71
+ printf '\n' >&2
72
+ fi
73
+ }
74
+
75
+ monitor_progress() {
76
+ local total=$1
77
+ local watch_path=$2
78
+ local start_avail=$3
79
+ local current_avail=0
80
+ local freed=0
81
+ local max_freed=0
82
+ local items=0
83
+ local last_render=0
84
+ local now msg
85
+
86
+ while true; do
87
+ if IFS= read -r -t "$progress_interval" msg; then
88
+ [ "$msg" != "__FAST_RM_DONE__" ] || break
89
+ items=$((items + 1))
90
+ fi
91
+
92
+ current_avail=$(fs_avail_bytes "$watch_path")
93
+ case "$current_avail" in
94
+ ''|*[!0-9]*) current_avail=$start_avail ;;
95
+ esac
96
+ freed=$((current_avail - start_avail))
97
+ [ "$freed" -ge 0 ] || freed=0
98
+ if [ "$freed" -gt "$max_freed" ]; then
99
+ max_freed=$freed
100
+ fi
101
+
102
+ now=$(date +%s)
103
+ if [ $((now - last_render)) -ge "$progress_interval" ]; then
104
+ progress_bar "$max_freed" "$total" "$items" 0
105
+ last_render=$now
106
+ fi
107
+ done
108
+
109
+ current_avail=$(fs_avail_bytes "$watch_path")
110
+ case "$current_avail" in
111
+ ''|*[!0-9]*) current_avail=$start_avail ;;
112
+ esac
113
+ freed=$((current_avail - start_avail))
114
+ [ "$freed" -ge 0 ] || freed=0
115
+ if [ "$freed" -gt "$max_freed" ]; then
116
+ max_freed=$freed
117
+ fi
118
+ progress_bar "$max_freed" "$total" "$items" 1
119
+ }
120
+
121
+ delete_stage() {
122
+ local min_depth=$1
123
+ local max_depth=$2
124
+
125
+ if [ "$progress" = 1 ]; then
126
+ find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \
127
+ | FAST_RM_PROGRESS_FIFO=$progress_fifo xargs -0 -r -n 1 -P "$jobs" bash -c '
128
+ set -e
129
+ for path do
130
+ rm -rf -- "$path"
131
+ printf "1\n" > "$FAST_RM_PROGRESS_FIFO" || true
132
+ done
133
+ ' _
134
+ else
135
+ find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \
136
+ | xargs -0 -r -n 1 -P "$jobs" rm -rf --
137
+ fi
138
+ }
139
+
140
+ [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
141
+
142
+ target=$1
143
+ jobs=${2:-${FAST_RM_JOBS:-$(nproc 2>/dev/null || echo 32)}}
144
+ shard_depth=${FAST_RM_SHARD_DEPTH:-3}
145
+ progress=${FAST_RM_PROGRESS:-1}
146
+ progress_interval=${FAST_RM_PROGRESS_INTERVAL:-1}
147
+ exact_total=${FAST_RM_EXACT_TOTAL:-0}
148
+
149
+ case "$jobs" in
150
+ ''|*[!0-9]*) die "JOBS must be a positive integer" ;;
151
+ esac
152
+ case "$shard_depth" in
153
+ ''|*[!0-9]*) die "FAST_RM_SHARD_DEPTH must be a positive integer" ;;
154
+ esac
155
+ case "$progress" in
156
+ 0|1) ;;
157
+ *) die "FAST_RM_PROGRESS must be 0 or 1" ;;
158
+ esac
159
+ case "$exact_total" in
160
+ 0|1) ;;
161
+ *) die "FAST_RM_EXACT_TOTAL must be 0 or 1" ;;
162
+ esac
163
+ case "$progress_interval" in
164
+ ''|*[!0-9]*) die "FAST_RM_PROGRESS_INTERVAL must be a positive integer" ;;
165
+ esac
166
+
167
+ [ "$jobs" -gt 0 ] || die "JOBS must be > 0"
168
+ [ "$shard_depth" -gt 0 ] || die "FAST_RM_SHARD_DEPTH must be > 0"
169
+ [ "$progress_interval" -gt 0 ] || die "FAST_RM_PROGRESS_INTERVAL must be > 0"
170
+
171
+ while [ "$target" != "/" ] && [ "${target%/}" != "$target" ]; do
172
+ target=${target%/}
173
+ done
174
+
175
+ [ -n "$target" ] || die "empty target"
176
+ [ "$target" != "/" ] || die "refuse to delete /"
177
+
178
+ parent=$(dirname -- "$target")
179
+ base=$(basename -- "$target")
180
+ [ "$base" != "." ] && [ "$base" != ".." ] || die "bad target: $target"
181
+
182
+ parent_abs=$(realpath -- "$parent")
183
+ target_abs=$parent_abs/$base
184
+
185
+ if [ ! -e "$target_abs" ] && [ ! -L "$target_abs" ]; then
186
+ echo "fast_rm: not found: $target_abs"
187
+ exit 0
188
+ fi
189
+
190
+ home_abs=$(realpath -- "${HOME:-/nonexistent}" 2>/dev/null || true)
191
+ case "$target_abs" in
192
+ "/"|"/cpfs01"|"/cpfs01/users"|"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong"|"/mnt"|"/mnt/workspace")
193
+ die "refuse to delete protected path: $target_abs"
194
+ ;;
195
+ esac
196
+ [ -z "$home_abs" ] || [ "$target_abs" != "$home_abs" ] || die "refuse to delete HOME: $target_abs"
197
+
198
+ mount_point=$(findmnt -T "$target_abs" -n -o TARGET 2>/dev/null || true)
199
+ [ -z "$mount_point" ] || [ "$mount_point" != "$target_abs" ] || die "refuse to delete mount point: $target_abs"
200
+
201
+ cwd=$(pwd -P)
202
+ case "$cwd/" in
203
+ "$target_abs"/*|"$target_abs/"*) die "refuse to delete current working tree: $target_abs" ;;
204
+ esac
205
+
206
+ if [ ! -d "$target_abs" ] || [ -L "$target_abs" ]; then
207
+ rm -f -- "$target_abs"
208
+ exit 0
209
+ fi
210
+
211
+ progress_fifo=
212
+ progress_pid=
213
+ progress_keepalive_fd=
214
+ cleanup_progress() {
215
+ if [ -n "$progress_pid" ]; then
216
+ kill "$progress_pid" 2>/dev/null || true
217
+ fi
218
+ if [ -n "$progress_keepalive_fd" ]; then
219
+ exec {progress_keepalive_fd}>&- 2>/dev/null || true
220
+ fi
221
+ if [ -n "$progress_fifo" ]; then
222
+ rm -f -- "$progress_fifo"
223
+ fi
224
+ }
225
+ trap cleanup_progress EXIT
226
+
227
+ safe_base=$(printf '%s' "$base" | tr -c 'A-Za-z0-9._-' '_')
228
+ if [[ "$base" == .deleting_* ]]; then
229
+ trash=$target_abs
230
+ else
231
+ trash=$parent_abs/.deleting_${safe_base}.$(date +%Y%m%d_%H%M%S).$$
232
+ mv -- "$target_abs" "$trash"
233
+ fi
234
+
235
+ echo "fast_rm: deleting $trash with jobs=$jobs shard_depth=$shard_depth"
236
+
237
+ if [ "$progress" = 1 ]; then
238
+ total_bytes=0
239
+ if [ "$exact_total" = 1 ]; then
240
+ echo "fast_rm: exact sizing target..." >&2
241
+ total_bytes=$(du -sB1 -- "$trash" 2>/dev/null | awk 'NR==1 {print $1}')
242
+ case "$total_bytes" in
243
+ ''|*[!0-9]*) total_bytes=0 ;;
244
+ esac
245
+ echo "fast_rm: total $(format_bytes "$total_bytes")" >&2
246
+ else
247
+ echo "fast_rm: skip exact sizing; showing freed space estimate" >&2
248
+ fi
249
+ start_avail=$(fs_avail_bytes "$parent_abs")
250
+ case "$start_avail" in
251
+ ''|*[!0-9]*) start_avail=0 ;;
252
+ esac
253
+ progress_fifo=$(mktemp -u "${TMPDIR:-/tmp}/fast_rm_progress.XXXXXX")
254
+ mkfifo "$progress_fifo"
255
+ start_ts=$(date +%s)
256
+ monitor_progress "$total_bytes" "$parent_abs" "$start_avail" < "$progress_fifo" &
257
+ progress_pid=$!
258
+ exec {progress_keepalive_fd}>"$progress_fifo"
259
+ fi
260
+
261
+ if [ "$shard_depth" -gt 1 ]; then
262
+ delete_stage "$shard_depth" "$shard_depth"
263
+ fi
264
+
265
+ delete_stage 1 1
266
+
267
+ rmdir -- "$trash" 2>/dev/null || rm -rf -- "$trash"
268
+
269
+ if [ "$progress" = 1 ]; then
270
+ printf '%s\n' "__FAST_RM_DONE__" >&$progress_keepalive_fd || true
271
+ wait "$progress_pid" 2>/dev/null || true
272
+ progress_pid=
273
+ exec {progress_keepalive_fd}>&- 2>/dev/null || true
274
+ progress_keepalive_fd=
275
+ rm -f -- "$progress_fifo"
276
+ progress_fifo=
277
+ fi
278
+
279
+ echo "fast_rm: done"
oss_download.sh ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 用cp只有几m每秒,特别慢,但是用api就特别快
2
+ # TODO: 封装成一个下载目录的脚本
3
+
4
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
5
+ # ./ossutil cp -r \
6
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/v11-20260202-204134/epoch=4-step=1600.ckpt \
7
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/v11-20260202-204134 \
8
+ # --parallel 32
9
+
10
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
11
+ # ./ossutil cp -r \
12
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/vae-20260123-085324/epoch=99-step=34000.ckpt \
13
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/vae-20260123-085324 \
14
+ # --parallel 32
15
+
16
+ # 这样下载好像有问题,会在文件末尾写入类似5.546607(s) elapsed的字符
17
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
18
+ # ./ossutil api get-object \
19
+ # --bucket cpfs-to-new-cpfs \
20
+ # --key VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/v11-20260202-204134/epoch=4-step=1600.ckpt/output_dir/model-00013-of-00013.safetensors \
21
+ # > /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/v11-20260202-204134/epoch=4-step=1600.ckpt/output_dir/model-00013-of-00013.safetensors
22
+
23
+ BUCKET="cpfs-to-new-cpfs"
24
+ OSS_PREFIX="VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/v15-20260224-013149-step3500-ae/"
25
+ LOCAL_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/wan/v15-20260224-013149-step3500-ae/"
26
+
27
+ mkdir -p "$LOCAL_DIR"
28
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
29
+
30
+ ./ossutil -c .ossutilconfig ls "oss://$BUCKET/$OSS_PREFIX" | grep -o "oss://$BUCKET/.*" | grep -v "/$" | while read -r OSS_URI; do
31
+ KEY="${OSS_URI#oss://$BUCKET/}"
32
+ RELATIVE_PATH="${KEY#$OSS_PREFIX}"
33
+ LOCAL_FILE="$LOCAL_DIR$RELATIVE_PATH"
34
+ LOCAL_SUBDIR=$(dirname "$LOCAL_FILE")
35
+
36
+ if [ ! -d "$LOCAL_SUBDIR" ]; then
37
+ mkdir -p "$LOCAL_SUBDIR"
38
+ fi
39
+
40
+ echo "downloading: $RELATIVE_PATH"
41
+
42
+ ./ossutil -c .ossutilconfig api get-object \
43
+ --bucket "$BUCKET" \
44
+ --key "$KEY" \
45
+ > "$LOCAL_FILE"
46
+
47
+ if [ $? -eq 0 ]; then
48
+ python3 -c "
49
+ import sys, re
50
+ with open(sys.argv[1], 'r+b') as f:
51
+ f.seek(0, 2)
52
+ size = f.tell()
53
+ if size == 0:
54
+ sys.exit(0)
55
+ chunk = min(4096, size)
56
+ f.seek(-chunk, 2)
57
+ tail = f.read()
58
+ m = re.search(br'(?:\s*[0-9]+\.[0-9]+\(s\) elapsed\s*)+$', tail)
59
+ if m:
60
+ f.truncate(size - len(m.group(0)))
61
+ " "$LOCAL_FILE"
62
+ echo "成功 -> $RELATIVE_PATH"
63
+ else
64
+ echo "失败 -> $KEY"
65
+ echo "失败 -> $RELATIVE_PATH"
66
+ fi
67
+ done
oss_upload.sh ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
2
+ # ./ossutil cp -r \
3
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/v6x/ \
4
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan
5
+
6
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
7
+ # ./ossutil cp -r \
8
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/2026011x/ \
9
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan
10
+
11
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
12
+ # ./ossutil cp -r \
13
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/loading_oss/ \
14
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan
15
+
16
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/ossutil-2.2.0-linux-amd64
17
+ ./ossutil -c .ossutilconfig \
18
+ cp -r \
19
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/datasets/HumanML3D_token \
20
+ oss://cpfs-to-new-cpfs/VideoGen/user_data/yetinghong/datasets/HumanML3D_token
21
+
22
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
23
+ # ./ossutil cp -r \
24
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_2858_pose_embed_v2/ \
25
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/dance_2858_pose_embed_v2
26
+
27
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/packages/ossutil-2.2.0-linux-amd64
28
+ # ./ossutil cp -r \
29
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/UniAnimate-DiT/dance_1723_new/ \
30
+ # oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong/code/UniAnimate-DiT/dance_1723_new
31
+
32
+
33
+
read_pt.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ # 1. 你的文件路径
4
+ file_path = "/cpfs01/videogen/gvhmr_smpl/part_10/--9DvmkocUk_00010_44_171/hmr4d_results.pt"
5
+
6
+ # 2. 加载数据
7
+ data = torch.load(file_path)
8
+
9
+ # 3. 指定要检查的四个 key
10
+ keys_to_check = ['smpl_params_global', 'smpl_params_incam', 'K_fullimg', 'net_outputs']
11
+
12
+ print(f"正在检查文件结构...")
13
+ print("=" * 50)
14
+
15
+ # 4. 开始遍历检查
16
+ for key in keys_to_check:
17
+ if key not in data:
18
+ print(f"❌ 找不到: {key}")
19
+ continue
20
+
21
+ val = data[key]
22
+
23
+ # 情况 A:直接是张量 (例如 K_fullimg)
24
+ if isinstance(val, torch.Tensor):
25
+ print(f"✅ 【{key}】 (直接是张量) \n └── 形状: {val.shape}")
26
+
27
+ # 情况 B:是嵌套字典 (例如 smpl_params_global, net_outputs)
28
+ elif isinstance(val, dict):
29
+ print(f"📦 【{key}】 (嵌套字典) 里面包含:")
30
+ for sub_k, sub_v in val.items():
31
+ if isinstance(sub_v, torch.Tensor):
32
+ print(f" ├── {sub_k}: {sub_v.shape}")
33
+ elif isinstance(sub_v, dict):
34
+ print(f" ├── {sub_k}: [又是一个字典] 包含 keys -> {list(sub_v.keys())}")
35
+ else:
36
+ print(f" ├── {sub_k}: [其他类型] {type(sub_v)}")
37
+
38
+ # 情况 C:其他未知类型
39
+ else:
40
+ print(f"❓ 【{key}】 (其他类型: {type(val)})")
41
+
42
+ print("-" * 50)
repair_cache_jsonl.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Repair cached confidence/reason fields in JSONL files in place."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ import sys
9
+ from collections import Counter
10
+ from pathlib import Path
11
+ from tempfile import NamedTemporaryFile
12
+ from typing import Dict, Iterable, Iterator, List, Tuple
13
+
14
+ ROOT = Path(__file__).resolve().parent.parent
15
+ if str(ROOT) not in sys.path:
16
+ sys.path.insert(0, str(ROOT))
17
+
18
+ from answer_utils import repair_confidence_reason_record_fields
19
+
20
+
21
+ def parse_args() -> argparse.Namespace:
22
+ parser = argparse.ArgumentParser(
23
+ description="Repair cache JSONL files by backfilling conf/reason from raw output."
24
+ )
25
+ parser.add_argument(
26
+ "paths",
27
+ nargs="*",
28
+ default=["cache/benchmarks"],
29
+ help="Files or directories to scan. Defaults to cache/benchmarks.",
30
+ )
31
+ parser.add_argument(
32
+ "--dry-run",
33
+ action="store_true",
34
+ help="Scan and report changes without rewriting files.",
35
+ )
36
+ parser.add_argument(
37
+ "--verbose",
38
+ action="store_true",
39
+ help="Print per-file repair counts.",
40
+ )
41
+ return parser.parse_args()
42
+
43
+
44
+ def iter_jsonl_files(paths: Iterable[str]) -> Iterator[Path]:
45
+ seen = set()
46
+ for raw_path in paths:
47
+ path = Path(raw_path)
48
+ if path.is_file() and path.suffix == ".jsonl":
49
+ resolved = path.resolve()
50
+ if resolved not in seen:
51
+ seen.add(resolved)
52
+ yield path
53
+ continue
54
+ if path.is_dir():
55
+ for item in sorted(path.rglob("*.jsonl")):
56
+ resolved = item.resolve()
57
+ if resolved in seen:
58
+ continue
59
+ seen.add(resolved)
60
+ yield item
61
+
62
+
63
+ def changed_field_counts(before: Dict, after: Dict) -> Counter:
64
+ counts: Counter = Counter()
65
+ keys = set(before.keys()) | set(after.keys())
66
+ for key in keys:
67
+ if before.get(key) == after.get(key):
68
+ continue
69
+ if key == "conf" or key.endswith("_conf"):
70
+ counts["conf_fields"] += 1
71
+ elif key == "reason" or key.endswith("_reason"):
72
+ counts["reason_fields"] += 1
73
+ else:
74
+ counts["other_fields"] += 1
75
+ return counts
76
+
77
+
78
+ def repair_file(path: Path, dry_run: bool = False) -> Tuple[bool, Counter]:
79
+ file_counts: Counter = Counter()
80
+ repaired_lines: List[str] = []
81
+
82
+ with path.open("r", encoding="utf-8") as handle:
83
+ for line_no, raw_line in enumerate(handle, start=1):
84
+ stripped = raw_line.strip()
85
+ if not stripped:
86
+ repaired_lines.append(raw_line)
87
+ continue
88
+ try:
89
+ record = json.loads(stripped)
90
+ except json.JSONDecodeError as exc:
91
+ raise ValueError(f"{path}:{line_no}: invalid json: {exc}") from exc
92
+
93
+ repaired = repair_confidence_reason_record_fields(record)
94
+ if repaired != record:
95
+ file_counts["records_changed"] += 1
96
+ file_counts.update(changed_field_counts(record, repaired))
97
+ repaired_lines.append(json.dumps(repaired, ensure_ascii=False) + "\n")
98
+
99
+ changed = bool(file_counts["records_changed"])
100
+ if changed and not dry_run:
101
+ with NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
102
+ tmp.writelines(repaired_lines)
103
+ tmp_path = Path(tmp.name)
104
+ tmp_path.replace(path)
105
+
106
+ return changed, file_counts
107
+
108
+
109
+ def main() -> int:
110
+ args = parse_args()
111
+ totals: Counter = Counter()
112
+ files = list(iter_jsonl_files(args.paths))
113
+
114
+ if not files:
115
+ print("No .jsonl files found.")
116
+ return 0
117
+
118
+ for path in files:
119
+ changed, file_counts = repair_file(path, dry_run=args.dry_run)
120
+ totals["files_scanned"] += 1
121
+ if changed:
122
+ totals["files_changed"] += 1
123
+ totals.update(file_counts)
124
+ if args.verbose and changed:
125
+ rel = path.as_posix()
126
+ print(
127
+ f"{rel}: records={file_counts['records_changed']} "
128
+ f"conf={file_counts['conf_fields']} reason={file_counts['reason_fields']}"
129
+ )
130
+
131
+ mode = "dry-run" if args.dry_run else "rewritten"
132
+ print(
133
+ f"[{mode}] files_scanned={totals['files_scanned']} "
134
+ f"files_changed={totals['files_changed']} "
135
+ f"records_changed={totals['records_changed']} "
136
+ f"conf_fields={totals['conf_fields']} "
137
+ f"reason_fields={totals['reason_fields']}"
138
+ )
139
+ return 0
140
+
141
+
142
+ if __name__ == "__main__":
143
+ raise SystemExit(main())
repair_math_cache_answers.py ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Repair math benchmark cache predictions/correct flags in place using current answer parsing."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ import sys
9
+ from collections import Counter, defaultdict
10
+ from pathlib import Path
11
+ from tempfile import NamedTemporaryFile
12
+ from typing import Any, Dict, Iterable, Iterator, List, Tuple
13
+
14
+ PROJECT_ROOT = Path(__file__).resolve().parents[1]
15
+ if str(PROJECT_ROOT) not in sys.path:
16
+ sys.path.insert(0, str(PROJECT_ROOT))
17
+
18
+ from answer_utils import (
19
+ check_answer,
20
+ extract_answer,
21
+ extract_confidence_reason,
22
+ repair_confidence_reason_record_fields,
23
+ )
24
+
25
+
26
+ def parse_args() -> argparse.Namespace:
27
+ parser = argparse.ArgumentParser(description=__doc__)
28
+ parser.add_argument(
29
+ "--root",
30
+ default="cache/benchmarks/math",
31
+ help="Math benchmark cache root to repair. Default: cache/benchmarks/math",
32
+ )
33
+ parser.add_argument(
34
+ "--dry-run",
35
+ action="store_true",
36
+ help="Report intended changes without rewriting files.",
37
+ )
38
+ parser.add_argument(
39
+ "--verbose",
40
+ action="store_true",
41
+ help="Print per-file change counts.",
42
+ )
43
+ return parser.parse_args()
44
+
45
+
46
+ def _safe_str(value: Any) -> str:
47
+ if value is None:
48
+ return ""
49
+ return str(value).strip()
50
+
51
+
52
+ def _iter_stage_files(root: Path) -> Iterator[Path]:
53
+ for path in sorted(root.rglob("*.stage*.cached.jsonl")):
54
+ if path.is_file():
55
+ yield path
56
+
57
+
58
+ def _iter_only_files(root: Path) -> Iterator[Path]:
59
+ for path in sorted(root.rglob("only/*/*.jsonl")):
60
+ if path.is_file():
61
+ yield path
62
+
63
+
64
+ def _write_lines_atomic(path: Path, lines: List[str]) -> None:
65
+ with NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
66
+ tmp.writelines(lines)
67
+ tmp_path = Path(tmp.name)
68
+ tmp_path.replace(path)
69
+
70
+
71
+ def _repair_stage_record(record: Dict[str, Any], counts: Counter) -> Dict[str, Any]:
72
+ updated = repair_confidence_reason_record_fields(record)
73
+ task = _safe_str(updated.get("task")).lower()
74
+ if task != "math":
75
+ return updated
76
+
77
+ gold = updated.get("label")
78
+ for stage_idx in (1, 2, 3):
79
+ raw_key = f"stage{stage_idx}_raw"
80
+ pred_key = f"stage{stage_idx}_pred"
81
+ correct_key = f"stage{stage_idx}_correct"
82
+ raw_text = _safe_str(updated.get(raw_key))
83
+ if raw_text:
84
+ new_pred = _safe_str(extract_answer(task, raw_text))
85
+ if new_pred != _safe_str(updated.get(pred_key)):
86
+ updated[pred_key] = new_pred
87
+ counts[f"stage{stage_idx}_pred_changed"] += 1
88
+ new_correct = bool(check_answer(task, new_pred, gold))
89
+ if bool(updated.get(correct_key, False)) != new_correct:
90
+ updated[correct_key] = new_correct
91
+ counts[f"stage{stage_idx}_correct_changed"] += 1
92
+
93
+ if stage_idx >= 3:
94
+ continue
95
+ retry_raw_key = f"stage{stage_idx}_retry_raw"
96
+ retry_pred_key = f"stage{stage_idx}_retry_pred"
97
+ retry_correct_key = f"stage{stage_idx}_retry_correct"
98
+ retry_raw = _safe_str(updated.get(retry_raw_key))
99
+ if retry_raw:
100
+ new_retry_pred = _safe_str(extract_answer(task, retry_raw))
101
+ if new_retry_pred != _safe_str(updated.get(retry_pred_key)):
102
+ updated[retry_pred_key] = new_retry_pred
103
+ counts[f"stage{stage_idx}_retry_pred_changed"] += 1
104
+ new_retry_correct = bool(check_answer(task, new_retry_pred, gold))
105
+ if bool(updated.get(retry_correct_key, False)) != new_retry_correct:
106
+ updated[retry_correct_key] = new_retry_correct
107
+ counts[f"stage{stage_idx}_retry_correct_changed"] += 1
108
+ return updated
109
+
110
+
111
+ def repair_stage_file(path: Path, dry_run: bool = False) -> Counter:
112
+ counts: Counter = Counter()
113
+ out_lines: List[str] = []
114
+ changed = False
115
+
116
+ with path.open("r", encoding="utf-8") as handle:
117
+ for raw_line in handle:
118
+ stripped = raw_line.strip()
119
+ if not stripped:
120
+ out_lines.append(raw_line)
121
+ continue
122
+ record = json.loads(stripped)
123
+ updated = _repair_stage_record(record, counts)
124
+ if updated != record:
125
+ changed = True
126
+ counts["records_changed"] += 1
127
+ out_lines.append(json.dumps(updated, ensure_ascii=False) + "\n")
128
+
129
+ if changed and not dry_run:
130
+ _write_lines_atomic(path, out_lines)
131
+ if changed:
132
+ counts["files_changed"] += 1
133
+ counts["files_scanned"] += 1
134
+ return counts
135
+
136
+
137
+ def _only_report_for_rows(path: Path, rows: List[Dict[str, Any]]) -> Dict[str, Any]:
138
+ total = len(rows)
139
+ correct = sum(int(bool(row.get("correct", False))) for row in rows)
140
+ report_path = path.with_suffix(".report.json")
141
+ existing = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
142
+ report = dict(existing)
143
+ report.update(
144
+ {
145
+ "path": str(path),
146
+ "split": path.stem,
147
+ "total": total,
148
+ "correct": correct,
149
+ "accuracy": (correct / total) if total else 0.0,
150
+ }
151
+ )
152
+ by_task = defaultdict(lambda: {"total": 0, "correct": 0})
153
+ for row in rows:
154
+ task = _safe_str(row.get("task"))
155
+ by_task[task]["total"] += 1
156
+ by_task[task]["correct"] += int(bool(row.get("correct", False)))
157
+ if existing.get("by_task"):
158
+ merged = {}
159
+ for task_name, stats in dict(existing["by_task"]).items():
160
+ merged[task_name] = dict(stats)
161
+ for task_name, stats in by_task.items():
162
+ merged.setdefault(task_name, {})
163
+ merged[task_name]["total"] = stats["total"]
164
+ merged[task_name]["correct"] = stats["correct"]
165
+ report["by_task"] = merged
166
+ else:
167
+ report["by_task"] = dict(by_task)
168
+ return report
169
+
170
+
171
+ def _repair_only_record(record: Dict[str, Any], counts: Counter) -> Dict[str, Any]:
172
+ updated = repair_confidence_reason_record_fields(record)
173
+ task = _safe_str(updated.get("task")).lower()
174
+ if task != "math":
175
+ return updated
176
+
177
+ raw_text = _safe_str(updated.get("raw"))
178
+ if raw_text:
179
+ new_pred = _safe_str(extract_answer(task, raw_text))
180
+ if new_pred != _safe_str(updated.get("pred")):
181
+ updated["pred"] = new_pred
182
+ counts["pred_changed"] += 1
183
+ conf, reason = extract_confidence_reason(raw_text)
184
+ if float(updated.get("conf", 0.0) or 0.0) != float(conf):
185
+ updated["conf"] = float(conf)
186
+ counts["conf_changed"] += 1
187
+ if _safe_str(updated.get("reason")) != _safe_str(reason):
188
+ updated["reason"] = _safe_str(reason)
189
+ counts["reason_changed"] += 1
190
+ new_correct = bool(check_answer(task, new_pred, updated.get("label")))
191
+ if bool(updated.get("correct", False)) != new_correct:
192
+ updated["correct"] = new_correct
193
+ counts["correct_changed"] += 1
194
+ return updated
195
+
196
+
197
+ def repair_only_file(path: Path, dry_run: bool = False) -> Counter:
198
+ counts: Counter = Counter()
199
+ out_lines: List[str] = []
200
+ changed = False
201
+ rows: List[Dict[str, Any]] = []
202
+
203
+ with path.open("r", encoding="utf-8") as handle:
204
+ for raw_line in handle:
205
+ stripped = raw_line.strip()
206
+ if not stripped:
207
+ out_lines.append(raw_line)
208
+ continue
209
+ record = json.loads(stripped)
210
+ updated = _repair_only_record(record, counts)
211
+ rows.append(updated)
212
+ if updated != record:
213
+ changed = True
214
+ counts["records_changed"] += 1
215
+ out_lines.append(json.dumps(updated, ensure_ascii=False) + "\n")
216
+
217
+ report = _only_report_for_rows(path, rows)
218
+ report_path = path.with_suffix(".report.json")
219
+ old_report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else None
220
+ report_changed = old_report != report
221
+
222
+ if (changed or report_changed) and not dry_run:
223
+ if changed:
224
+ _write_lines_atomic(path, out_lines)
225
+ report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
226
+
227
+ if changed:
228
+ counts["files_changed"] += 1
229
+ if report_changed:
230
+ counts["reports_changed"] += 1
231
+ counts["files_scanned"] += 1
232
+ return counts
233
+
234
+
235
+ def main() -> int:
236
+ args = parse_args()
237
+ root = Path(args.root)
238
+ if not root.exists():
239
+ raise FileNotFoundError(f"Root does not exist: {root}")
240
+
241
+ totals: Counter = Counter()
242
+
243
+ for path in _iter_stage_files(root):
244
+ file_counts = repair_stage_file(path, dry_run=args.dry_run)
245
+ totals.update(file_counts)
246
+ if args.verbose and (file_counts["records_changed"] or file_counts["files_changed"]):
247
+ print(f"{path}: stage_records_changed={file_counts['records_changed']}")
248
+
249
+ for path in _iter_only_files(root):
250
+ file_counts = repair_only_file(path, dry_run=args.dry_run)
251
+ totals.update(file_counts)
252
+ if args.verbose and (file_counts["records_changed"] or file_counts["reports_changed"]):
253
+ print(
254
+ f"{path}: only_records_changed={file_counts['records_changed']} "
255
+ f"reports_changed={file_counts['reports_changed']}"
256
+ )
257
+
258
+ mode = "dry-run" if args.dry_run else "rewritten"
259
+ print(
260
+ f"[{mode}] files_scanned={totals['files_scanned']} files_changed={totals['files_changed']} "
261
+ f"reports_changed={totals['reports_changed']} records_changed={totals['records_changed']}"
262
+ )
263
+ for key in sorted(k for k in totals if k not in {"files_scanned", "files_changed", "reports_changed", "records_changed"}):
264
+ print(f"{key}={totals[key]}")
265
+ return 0
266
+
267
+
268
+ if __name__ == "__main__":
269
+ raise SystemExit(main())
report_stage_costs.py ADDED
@@ -0,0 +1,622 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Report per-stage cascade/only cost and accuracy statistics from benchmark caches."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ import os
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Dict, Iterable, List, Optional, Sequence, Tuple
12
+
13
+ PROJECT_ROOT = Path(__file__).resolve().parents[1]
14
+ if str(PROJECT_ROOT) not in sys.path:
15
+ sys.path.insert(0, str(PROJECT_ROOT))
16
+
17
+ from transformers import AutoTokenizer
18
+
19
+ from answer_utils import check_answer, repair_confidence_reason_record_fields, task_instruction
20
+ from cascade_utils import (
21
+ build_router_prompt_for_cost,
22
+ count_tokens,
23
+ iter_stage_indices,
24
+ render_chat,
25
+ safe_str,
26
+ stage_correct_key,
27
+ stage_output_exists,
28
+ stage_pred_key,
29
+ stage_raw_key,
30
+ stage_retry_correct_key,
31
+ stage_retry_output_exists,
32
+ stage_retry_pred_key,
33
+ stage_retry_raw_key,
34
+ )
35
+ from config import MODEL_GROUPS, SYSTEM_MESSAGE_DEFAULT
36
+
37
+
38
+ def _read_jsonl(path: Path) -> List[Dict]:
39
+ rows: List[Dict] = []
40
+ with path.open("r", encoding="utf-8") as handle:
41
+ for line in handle:
42
+ line = line.strip()
43
+ if line:
44
+ rows.append(repair_confidence_reason_record_fields(json.loads(line)))
45
+ return rows
46
+
47
+
48
+ def _price_per_token(price_per_million: float) -> float:
49
+ return float(price_per_million) / 1_000_000.0
50
+
51
+
52
+ def _parse_tokenizer_overrides(values: Sequence[str]) -> Dict[Tuple[str, str], str]:
53
+ overrides: Dict[Tuple[str, str], str] = {}
54
+ for value in values:
55
+ raw = safe_str(value)
56
+ if not raw:
57
+ continue
58
+ if "=" not in raw or ":" not in raw.split("=", 1)[0]:
59
+ raise ValueError(
60
+ f"Invalid --tokenizer-source {value!r}. Expected format: <group>:<stage>=<source>"
61
+ )
62
+ key_part, source = raw.split("=", 1)
63
+ group, stage = key_part.split(":", 1)
64
+ group = safe_str(group).lower()
65
+ stage = safe_str(stage).lower()
66
+ source = safe_str(source)
67
+ if not group or not stage or not source:
68
+ raise ValueError(
69
+ f"Invalid --tokenizer-source {value!r}. Expected format: <group>:<stage>=<source>"
70
+ )
71
+ overrides[(group, stage)] = source
72
+ return overrides
73
+
74
+
75
+ def _resolve_tokenizer_source(
76
+ stage_cfg: Dict,
77
+ group: str,
78
+ stage_name: str,
79
+ overrides: Dict[Tuple[str, str], str],
80
+ ) -> Tuple[str, bool]:
81
+ override = overrides.get((group, stage_name.lower()))
82
+ if override:
83
+ return override, os.path.isdir(override)
84
+
85
+ candidates = [
86
+ safe_str(stage_cfg.get("tokenizer_path")),
87
+ safe_str(stage_cfg.get("model_path")),
88
+ safe_str(stage_cfg.get("model_name")),
89
+ ]
90
+ for candidate in candidates:
91
+ if candidate and os.path.isdir(candidate):
92
+ return candidate, True
93
+ for candidate in candidates:
94
+ if candidate and not os.path.isabs(candidate) and ("/" in candidate or Path(candidate).exists()):
95
+ return candidate, False
96
+
97
+ raise FileNotFoundError(
98
+ "No usable tokenizer source found for "
99
+ f"group={group} stage={stage_name}. "
100
+ f"Checked candidates={candidates}. "
101
+ "Pass --tokenizer-source <group>:<stage>=<source> to override."
102
+ )
103
+
104
+
105
+ def _load_tokenizer(
106
+ stage_cfg: Dict,
107
+ group: str,
108
+ stage_name: str,
109
+ overrides: Dict[Tuple[str, str], str],
110
+ ):
111
+ source, local_files_only = _resolve_tokenizer_source(stage_cfg, group, stage_name, overrides)
112
+ tokenizer = AutoTokenizer.from_pretrained(
113
+ source,
114
+ trust_remote_code=True,
115
+ local_files_only=local_files_only,
116
+ use_fast=False,
117
+ )
118
+ tokenizer.padding_side = "left"
119
+ if tokenizer.pad_token is None and tokenizer.eos_token is not None:
120
+ tokenizer.pad_token = tokenizer.eos_token
121
+ return tokenizer
122
+
123
+
124
+ def _compute_stage_cost(sample: Dict, stage_cfg: Dict, stage_index: int, tokenizer, retry: bool = False) -> float:
125
+ task_name = safe_str(sample.get("task"))
126
+ question = safe_str(sample.get("headline"))
127
+ rendered = render_chat(
128
+ tokenizer,
129
+ safe_str(stage_cfg.get("system_message")) or SYSTEM_MESSAGE_DEFAULT,
130
+ build_router_prompt_for_cost(question, task_name, stage_index, sample, retry=retry),
131
+ )
132
+ if retry:
133
+ raw_text = safe_str(sample.get(stage_retry_raw_key(stage_index))) or safe_str(sample.get(stage_retry_pred_key(stage_index)))
134
+ else:
135
+ raw_text = safe_str(sample.get(stage_raw_key(stage_index))) or safe_str(sample.get(stage_pred_key(stage_index)))
136
+ input_unit = _price_per_token(float(stage_cfg["input_cost_per_mtoken"]))
137
+ output_unit = _price_per_token(float(stage_cfg["output_cost_per_mtoken"]))
138
+ return count_tokens(tokenizer, rendered) * input_unit + count_tokens(tokenizer, raw_text) * output_unit
139
+
140
+
141
+ def _build_only_prompt(question: str, task_name: str) -> str:
142
+ return f"{task_instruction(task_name)}\n\nQuestion:\n{question}\n"
143
+
144
+
145
+ def _compute_only_cost(sample: Dict, stage_cfg: Dict, tokenizer) -> float:
146
+ task_name = safe_str(sample.get("task"))
147
+ question = safe_str(sample.get("headline"))
148
+ raw_text = safe_str(sample.get("raw")) or safe_str(sample.get("pred"))
149
+ rendered = render_chat(
150
+ tokenizer,
151
+ safe_str(stage_cfg.get("system_message")) or SYSTEM_MESSAGE_DEFAULT,
152
+ _build_only_prompt(question, task_name),
153
+ )
154
+ input_unit = _price_per_token(float(stage_cfg["input_cost_per_mtoken"]))
155
+ output_unit = _price_per_token(float(stage_cfg["output_cost_per_mtoken"]))
156
+ return count_tokens(tokenizer, rendered) * input_unit + count_tokens(tokenizer, raw_text) * output_unit
157
+
158
+
159
+ def _empty_stats(applicable: bool = True) -> Dict[str, Optional[float]]:
160
+ return {
161
+ "applicable": bool(applicable),
162
+ "samples": 0,
163
+ "correct": 0,
164
+ "accuracy": None,
165
+ "total_cost": 0.0,
166
+ "avg_cost": None,
167
+ }
168
+
169
+
170
+ def _finalize_stats(stats: Dict[str, Optional[float]]) -> Dict[str, Optional[float]]:
171
+ samples = int(stats["samples"])
172
+ if not samples:
173
+ stats["accuracy"] = None
174
+ stats["avg_cost"] = None
175
+ return stats
176
+ stats["accuracy"] = float(stats["correct"]) / samples
177
+ stats["avg_cost"] = float(stats["total_cost"]) / samples
178
+ return stats
179
+
180
+
181
+ def _stage_slot_name(stage_index: int) -> str:
182
+ return f"stage{stage_index + 1}"
183
+
184
+
185
+ def _init_stage_summary() -> Dict[str, Dict[str, Dict[str, Optional[float]]]]:
186
+ summary: Dict[str, Dict[str, Dict[str, Optional[float]]]] = {}
187
+ for stage_index in iter_stage_indices():
188
+ stage_name = _stage_slot_name(stage_index)
189
+ summary[stage_name] = {
190
+ "first": _empty_stats(applicable=True),
191
+ "rerun": _empty_stats(applicable=(stage_index < 2)),
192
+ }
193
+ return summary
194
+
195
+
196
+ def _merge_stats(dst: Dict[str, Optional[float]], src: Dict[str, Optional[float]]) -> None:
197
+ dst["samples"] = int(dst["samples"]) + int(src["samples"])
198
+ dst["correct"] = int(dst["correct"]) + int(src["correct"])
199
+ dst["total_cost"] = float(dst["total_cost"]) + float(src["total_cost"])
200
+
201
+
202
+ def _iter_cascade_targets(
203
+ root: Path,
204
+ benchmarks: Sequence[str],
205
+ groups: Sequence[str],
206
+ splits: Sequence[str],
207
+ ) -> Iterable[Tuple[str, str, str, Path]]:
208
+ for benchmark in benchmarks:
209
+ for group in groups:
210
+ for split in splits:
211
+ path = root / benchmark / group / f"{split}.stage3.cached.jsonl"
212
+ if path.exists():
213
+ yield benchmark, group, split, path
214
+
215
+
216
+ def _iter_only_targets(
217
+ root: Path,
218
+ benchmarks: Sequence[str],
219
+ splits: Sequence[str],
220
+ ) -> Iterable[Tuple[str, str, str, Path]]:
221
+ for benchmark in benchmarks:
222
+ only_dir = root / benchmark / "only"
223
+ if not only_dir.exists():
224
+ continue
225
+ for model_dir in sorted(path for path in only_dir.iterdir() if path.is_dir()):
226
+ for split in splits:
227
+ path = model_dir / f"{split}.jsonl"
228
+ if path.exists():
229
+ yield benchmark, model_dir.name, split, path
230
+
231
+
232
+ def _build_cascade_rows(
233
+ root: Path,
234
+ benchmarks: Sequence[str],
235
+ groups: Sequence[str],
236
+ splits: Sequence[str],
237
+ tokenizer_overrides: Dict[Tuple[str, str], str],
238
+ recompute_costs: bool,
239
+ ) -> List[Dict]:
240
+ tokenizer_cache: Dict[Tuple[str, str], object] = {}
241
+ rows: List[Dict] = []
242
+ aggregate: Dict[Tuple[str, str], Dict] = {}
243
+
244
+ for benchmark, group, split, path in _iter_cascade_targets(root, benchmarks, groups, splits):
245
+ stage_cfgs = MODEL_GROUPS[group]["stages"]
246
+ samples = _read_jsonl(path)
247
+ if not samples:
248
+ continue
249
+
250
+ stage_summary = _init_stage_summary()
251
+ for sample in samples:
252
+ for stage_index in iter_stage_indices():
253
+ stage_name = _stage_slot_name(stage_index)
254
+ if stage_output_exists(sample, stage_index):
255
+ stats = stage_summary[stage_name]["first"]
256
+ stats["samples"] = int(stats["samples"]) + 1
257
+ stats["correct"] = int(stats["correct"]) + int(bool(sample.get(stage_correct_key(stage_index), False)))
258
+ cost_key = f"{stage_name}_cost"
259
+ if not recompute_costs and cost_key in sample:
260
+ cost_value = float(sample.get(cost_key, 0.0))
261
+ else:
262
+ tok_key = (group, stage_name)
263
+ if tok_key not in tokenizer_cache:
264
+ tokenizer_cache[tok_key] = _load_tokenizer(stage_cfgs[stage_index], group, stage_name, tokenizer_overrides)
265
+ cost_value = _compute_stage_cost(
266
+ sample, stage_cfgs[stage_index], stage_index, tokenizer_cache[tok_key], retry=False
267
+ )
268
+ stats["total_cost"] = float(stats["total_cost"]) + cost_value
269
+
270
+ if stage_index < 2 and stage_retry_output_exists(sample, stage_index):
271
+ stats = stage_summary[stage_name]["rerun"]
272
+ stats["samples"] = int(stats["samples"]) + 1
273
+ stats["correct"] = int(stats["correct"]) + int(bool(sample.get(stage_retry_correct_key(stage_index), False)))
274
+ retry_cost_key = f"{stage_name}_retry_cost"
275
+ if not recompute_costs and retry_cost_key in sample:
276
+ cost_value = float(sample.get(retry_cost_key, 0.0))
277
+ else:
278
+ tok_key = (group, stage_name)
279
+ if tok_key not in tokenizer_cache:
280
+ tokenizer_cache[tok_key] = _load_tokenizer(stage_cfgs[stage_index], group, stage_name, tokenizer_overrides)
281
+ cost_value = _compute_stage_cost(
282
+ sample, stage_cfgs[stage_index], stage_index, tokenizer_cache[tok_key], retry=True
283
+ )
284
+ stats["total_cost"] = float(stats["total_cost"]) + cost_value
285
+
286
+ for stage_name, slot_map in stage_summary.items():
287
+ for attempt_name, stats in slot_map.items():
288
+ _finalize_stats(stats)
289
+
290
+ row = {
291
+ "kind": "cascade",
292
+ "benchmark": benchmark,
293
+ "group": group,
294
+ "split": split,
295
+ "samples": len(samples),
296
+ "combo": safe_str(MODEL_GROUPS[group].get("display_name")) or group,
297
+ "stage_models": {
298
+ _stage_slot_name(idx): safe_str(stage_cfgs[idx].get("cache_id") or stage_cfgs[idx].get("model_name"))
299
+ for idx in iter_stage_indices()
300
+ },
301
+ "stages": stage_summary,
302
+ }
303
+ rows.append(row)
304
+
305
+ agg_key = (benchmark, group)
306
+ bucket = aggregate.setdefault(
307
+ agg_key,
308
+ {
309
+ "samples": 0,
310
+ "stage_models": row["stage_models"],
311
+ "combo": row["combo"],
312
+ "stages": _init_stage_summary(),
313
+ },
314
+ )
315
+ bucket["samples"] += len(samples)
316
+ for stage_name, slot_map in stage_summary.items():
317
+ for attempt_name, stats in slot_map.items():
318
+ _merge_stats(bucket["stages"][stage_name][attempt_name], stats)
319
+
320
+ final_rows: List[Dict] = []
321
+ for benchmark, group in sorted(aggregate):
322
+ bucket = aggregate[(benchmark, group)]
323
+ for stage_name, slot_map in bucket["stages"].items():
324
+ for attempt_name, stats in slot_map.items():
325
+ _finalize_stats(stats)
326
+ final_rows.append(
327
+ {
328
+ "kind": "cascade",
329
+ "benchmark": benchmark,
330
+ "group": group,
331
+ "split": "all",
332
+ "samples": bucket["samples"],
333
+ "combo": bucket["combo"],
334
+ "stage_models": bucket["stage_models"],
335
+ "stages": bucket["stages"],
336
+ }
337
+ )
338
+
339
+ rows.sort(key=lambda item: (item["benchmark"], item["group"], item["split"]))
340
+ final_rows.extend(rows)
341
+ return final_rows
342
+
343
+
344
+ def _load_only_report(path: Path) -> Optional[Dict]:
345
+ report_path = path.with_suffix(".report.json")
346
+ if not report_path.exists():
347
+ return None
348
+ return json.loads(report_path.read_text(encoding="utf-8"))
349
+
350
+
351
+ def _resolve_only_model_specs() -> Dict[str, Tuple[str, str, Dict]]:
352
+ specs: Dict[str, Tuple[str, str, Dict]] = {}
353
+ for group, group_cfg in MODEL_GROUPS.items():
354
+ for stage_index, stage_cfg in enumerate(group_cfg["stages"]):
355
+ model_id = safe_str(stage_cfg.get("cache_id"))
356
+ if not model_id:
357
+ continue
358
+ stage_name = _stage_slot_name(stage_index)
359
+ model_key = model_id.lower()
360
+ existing = specs.get(model_key)
361
+ if existing is None:
362
+ specs[model_key] = (group, stage_name, dict(stage_cfg))
363
+ continue
364
+ existing_group, existing_stage_name, existing_cfg = existing
365
+ comparable_keys = (
366
+ "tokenizer_path",
367
+ "model_path",
368
+ "model_name",
369
+ "input_cost_per_mtoken",
370
+ "output_cost_per_mtoken",
371
+ "system_message",
372
+ )
373
+ if any(safe_str(existing_cfg.get(key)) != safe_str(stage_cfg.get(key)) for key in comparable_keys):
374
+ raise ValueError(
375
+ f"Conflicting only model config for {model_id}: "
376
+ f"{existing_group}/{existing_stage_name} vs {group}/{stage_name}"
377
+ )
378
+ return specs
379
+
380
+
381
+ def _build_only_rows(
382
+ root: Path,
383
+ benchmarks: Sequence[str],
384
+ splits: Sequence[str],
385
+ tokenizer_overrides: Dict[Tuple[str, str], str],
386
+ ) -> List[Dict]:
387
+ model_specs = _resolve_only_model_specs()
388
+ tokenizer_cache: Dict[Tuple[str, str], object] = {}
389
+ rows: List[Dict] = []
390
+ aggregate: Dict[Tuple[str, str], Dict[str, float]] = {}
391
+
392
+ for benchmark, model_id, split, path in _iter_only_targets(root, benchmarks, splits):
393
+ samples = _read_jsonl(path)
394
+ if not samples:
395
+ continue
396
+
397
+ model_key = model_id.lower()
398
+ if model_key not in model_specs:
399
+ raise KeyError(f"Missing current config for only model: {model_id}")
400
+ group, stage_name, stage_cfg = model_specs[model_key]
401
+ tok_key = (group, stage_name)
402
+ if tok_key not in tokenizer_cache:
403
+ tokenizer_cache[tok_key] = _load_tokenizer(stage_cfg, group, stage_name, tokenizer_overrides)
404
+
405
+ total = len(samples)
406
+ correct = 0
407
+ total_cost = 0.0
408
+ for sample in samples:
409
+ pred = safe_str(sample.get("pred"))
410
+ gold = sample.get("label")
411
+ task_name = safe_str(sample.get("task"))
412
+ correct += int(check_answer(task_name, pred, gold))
413
+ total_cost += _compute_only_cost(sample, stage_cfg, tokenizer_cache[tok_key])
414
+
415
+ accuracy = (correct / total) if total > 0 else None
416
+ avg_cost = (total_cost / total) if total > 0 else None
417
+ model_name = safe_str(samples[0].get("model_name")) or safe_str(stage_cfg.get("model_name")) or model_id
418
+
419
+ row = {
420
+ "kind": "only",
421
+ "benchmark": benchmark,
422
+ "group": "only",
423
+ "split": split,
424
+ "samples": total,
425
+ "model_id": model_id,
426
+ "model_name": model_name,
427
+ "correct": correct,
428
+ "accuracy": accuracy,
429
+ "total_cost": total_cost,
430
+ "avg_cost": avg_cost,
431
+ }
432
+ rows.append(row)
433
+
434
+ agg_key = (benchmark, model_id)
435
+ bucket = aggregate.setdefault(
436
+ agg_key,
437
+ {
438
+ "samples": 0,
439
+ "correct": 0,
440
+ "total_cost": 0.0,
441
+ "model_name": model_name,
442
+ },
443
+ )
444
+ bucket["samples"] += total
445
+ bucket["correct"] += correct
446
+ bucket["total_cost"] += total_cost
447
+
448
+ final_rows: List[Dict] = []
449
+ for benchmark, model_id in sorted(aggregate):
450
+ bucket = aggregate[(benchmark, model_id)]
451
+ samples = int(bucket["samples"])
452
+ accuracy = (int(bucket["correct"]) / samples) if samples > 0 else None
453
+ avg_cost = (float(bucket["total_cost"]) / samples) if samples > 0 else None
454
+ final_rows.append(
455
+ {
456
+ "kind": "only",
457
+ "benchmark": benchmark,
458
+ "group": "only",
459
+ "split": "all",
460
+ "samples": samples,
461
+ "model_id": model_id,
462
+ "model_name": bucket["model_name"],
463
+ "correct": int(bucket["correct"]),
464
+ "accuracy": accuracy,
465
+ "total_cost": float(bucket["total_cost"]),
466
+ "avg_cost": avg_cost,
467
+ }
468
+ )
469
+
470
+ rows.sort(key=lambda item: (item["benchmark"], item["model_id"], item["split"]))
471
+ final_rows.extend(rows)
472
+ return final_rows
473
+
474
+
475
+ def _print_cascade_table(rows: Sequence[Dict]) -> None:
476
+ headers = [
477
+ "benchmark",
478
+ "group",
479
+ "split",
480
+ "samples",
481
+ "s1_first_acc",
482
+ "s1_rerun_acc",
483
+ "s2_first_acc",
484
+ "s2_rerun_acc",
485
+ "s3_first_acc",
486
+ "s3_rerun_acc",
487
+ "s1_first_cost",
488
+ "s1_rerun_cost",
489
+ "s2_first_cost",
490
+ "s2_rerun_cost",
491
+ "s3_first_cost",
492
+ "s3_rerun_cost",
493
+ ]
494
+ str_rows: List[Dict[str, str]] = []
495
+ for row in rows:
496
+ if row.get("kind") != "cascade":
497
+ continue
498
+ stages = row["stages"]
499
+ str_rows.append(
500
+ {
501
+ "benchmark": str(row["benchmark"]),
502
+ "group": str(row["group"]),
503
+ "split": str(row["split"]),
504
+ "samples": str(row["samples"]),
505
+ "s1_first_acc": _fmt(stages["stage1"]["first"]["accuracy"]),
506
+ "s1_rerun_acc": _fmt(stages["stage1"]["rerun"]["accuracy"]),
507
+ "s2_first_acc": _fmt(stages["stage2"]["first"]["accuracy"]),
508
+ "s2_rerun_acc": _fmt(stages["stage2"]["rerun"]["accuracy"]),
509
+ "s3_first_acc": _fmt(stages["stage3"]["first"]["accuracy"]),
510
+ "s3_rerun_acc": _fmt(stages["stage3"]["rerun"]["accuracy"]),
511
+ "s1_first_cost": _fmt(stages["stage1"]["first"]["avg_cost"]),
512
+ "s1_rerun_cost": _fmt(stages["stage1"]["rerun"]["avg_cost"]),
513
+ "s2_first_cost": _fmt(stages["stage2"]["first"]["avg_cost"]),
514
+ "s2_rerun_cost": _fmt(stages["stage2"]["rerun"]["avg_cost"]),
515
+ "s3_first_cost": _fmt(stages["stage3"]["first"]["avg_cost"]),
516
+ "s3_rerun_cost": _fmt(stages["stage3"]["rerun"]["avg_cost"]),
517
+ }
518
+ )
519
+ if not str_rows:
520
+ return
521
+ widths = {
522
+ header: max(len(header), *(len(item[header]) for item in str_rows)) if str_rows else len(header)
523
+ for header in headers
524
+ }
525
+ print(" ".join(header.ljust(widths[header]) for header in headers))
526
+ print(" ".join("-" * widths[header] for header in headers))
527
+ for row in str_rows:
528
+ print(" ".join(row[header].ljust(widths[header]) for header in headers))
529
+
530
+
531
+ def _print_only_table(rows: Sequence[Dict]) -> None:
532
+ headers = ["benchmark", "model_id", "split", "samples", "accuracy", "avg_cost"]
533
+ str_rows: List[Dict[str, str]] = []
534
+ for row in rows:
535
+ if row.get("kind") != "only":
536
+ continue
537
+ str_rows.append(
538
+ {
539
+ "benchmark": str(row["benchmark"]),
540
+ "model_id": str(row["model_id"]),
541
+ "split": str(row["split"]),
542
+ "samples": str(row["samples"]),
543
+ "accuracy": _fmt(row["accuracy"]),
544
+ "avg_cost": _fmt(row["avg_cost"]),
545
+ }
546
+ )
547
+ if not str_rows:
548
+ return
549
+ print()
550
+ widths = {
551
+ header: max(len(header), *(len(item[header]) for item in str_rows)) if str_rows else len(header)
552
+ for header in headers
553
+ }
554
+ print(" ".join(header.ljust(widths[header]) for header in headers))
555
+ print(" ".join("-" * widths[header] for header in headers))
556
+ for row in str_rows:
557
+ print(" ".join(row[header].ljust(widths[header]) for header in headers))
558
+
559
+
560
+ def _fmt(value: Optional[float]) -> str:
561
+ if value is None:
562
+ return "-"
563
+ return f"{float(value):.8f}"
564
+
565
+
566
+ def main() -> int:
567
+ parser = argparse.ArgumentParser(description=__doc__)
568
+ parser.add_argument("--root", default="cache/benchmarks", help="Benchmark cache root directory.")
569
+ parser.add_argument("--benchmarks", nargs="*", help="Benchmarks to include. Default: all under cache root.")
570
+ parser.add_argument("--groups", nargs="*", choices=sorted(MODEL_GROUPS), help="Cascade model groups to include. Default: all.")
571
+ parser.add_argument("--splits", nargs="*", default=["train", "test"], help="Splits to include. Default: train test.")
572
+ parser.add_argument(
573
+ "--tokenizer-source",
574
+ action="append",
575
+ default=[],
576
+ help="Override tokenizer source. Format: <group>:<stage>=<source>, e.g. qwen:stage1=/path/to/tokenizer",
577
+ )
578
+ parser.add_argument(
579
+ "--recompute-costs",
580
+ action="store_true",
581
+ help="Ignore stored cost fields and recompute costs from tokenizer/pricing config.",
582
+ )
583
+ parser.add_argument("--output-json", help="Optional path to write JSON results.")
584
+ args = parser.parse_args()
585
+
586
+ root = Path(args.root)
587
+ if not root.exists():
588
+ raise FileNotFoundError(f"Benchmark cache root does not exist: {root}")
589
+
590
+ benchmarks = list(args.benchmarks) if args.benchmarks else sorted(
591
+ path.name for path in root.iterdir() if path.is_dir() and path.name != "only"
592
+ )
593
+ groups = list(args.groups) if args.groups else sorted(MODEL_GROUPS)
594
+ splits = [str(split).strip() for split in args.splits if str(split).strip()]
595
+ tokenizer_overrides = _parse_tokenizer_overrides(args.tokenizer_source)
596
+
597
+ cascade_rows = _build_cascade_rows(root, benchmarks, groups, splits, tokenizer_overrides, args.recompute_costs)
598
+ only_rows = _build_only_rows(root, benchmarks, splits, tokenizer_overrides)
599
+ final_rows = cascade_rows + only_rows
600
+
601
+ _print_cascade_table(cascade_rows)
602
+ _print_only_table(only_rows)
603
+
604
+ if args.output_json:
605
+ output_path = Path(args.output_json)
606
+ output_path.parent.mkdir(parents=True, exist_ok=True)
607
+ payload = {
608
+ "root": str(root),
609
+ "benchmarks": benchmarks,
610
+ "groups": groups,
611
+ "splits": splits,
612
+ "cascade_results": cascade_rows,
613
+ "only_results": only_rows,
614
+ "results": final_rows,
615
+ }
616
+ output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
617
+
618
+ return 0
619
+
620
+
621
+ if __name__ == "__main__":
622
+ raise SystemExit(main())
report_stage_recovery.py ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Report conditional correction rates for cascade upgrades versus only models."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
11
+
12
+ PROJECT_ROOT = Path(__file__).resolve().parents[1]
13
+ if str(PROJECT_ROOT) not in sys.path:
14
+ sys.path.insert(0, str(PROJECT_ROOT))
15
+
16
+ from answer_utils import check_answer
17
+ from config import MODEL_GROUPS
18
+
19
+
20
+ def _safe_str(value: Any) -> str:
21
+ if value is None:
22
+ return ""
23
+ return str(value).strip()
24
+
25
+
26
+ def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
27
+ rows: List[Dict[str, Any]] = []
28
+ with path.open("r", encoding="utf-8") as handle:
29
+ for line in handle:
30
+ line = line.strip()
31
+ if line:
32
+ rows.append(json.loads(line))
33
+ return rows
34
+
35
+
36
+ def _cache_key(row: Dict[str, Any], fallback_index: int) -> int:
37
+ raw_key = row.get("_cache_index")
38
+ if raw_key is None:
39
+ return int(fallback_index)
40
+ return int(raw_key)
41
+
42
+
43
+ def _bool_correct(task: str, pred: Any, gold: Any) -> bool:
44
+ return bool(check_answer(task, _safe_str(pred), gold))
45
+
46
+
47
+ def _stage_pred(row: Dict[str, Any], stage_index: int) -> str:
48
+ return _safe_str(row.get(f"stage{stage_index + 1}_pred"))
49
+
50
+
51
+ def _stage_correct(row: Dict[str, Any], stage_index: int) -> bool:
52
+ return _bool_correct(_safe_str(row.get("task")), _stage_pred(row, stage_index), row.get("label"))
53
+
54
+
55
+ def _only_pred(row: Dict[str, Any]) -> str:
56
+ return _safe_str(row.get("pred"))
57
+
58
+
59
+ def _only_correct(row: Dict[str, Any]) -> bool:
60
+ return _bool_correct(_safe_str(row.get("task")), _only_pred(row), row.get("label"))
61
+
62
+
63
+ def _iter_benchmarks(root: Path, requested: Optional[Sequence[str]]) -> List[str]:
64
+ if requested:
65
+ return [str(name).strip() for name in requested if str(name).strip()]
66
+ return sorted(path.name for path in root.iterdir() if path.is_dir())
67
+
68
+
69
+ def _load_stage_rows(root: Path, benchmark: str, group: str, split: str) -> List[Dict[str, Any]]:
70
+ path = root / benchmark / group / f"{split}.stage3.cached.jsonl"
71
+ if not path.exists():
72
+ raise FileNotFoundError(path)
73
+ return _read_jsonl(path)
74
+
75
+
76
+ def _load_only_rows(root: Path, benchmark: str, model_id: str, split: str) -> Dict[int, Dict[str, Any]]:
77
+ path = root / benchmark / "only" / model_id.lower() / f"{split}.jsonl"
78
+ if not path.exists():
79
+ raise FileNotFoundError(path)
80
+ rows = _read_jsonl(path)
81
+ indexed: Dict[int, Dict[str, Any]] = {}
82
+ for fallback_index, row in enumerate(rows):
83
+ indexed[_cache_key(row, fallback_index)] = row
84
+ return indexed
85
+
86
+
87
+ def _make_bucket() -> Dict[str, Any]:
88
+ return {
89
+ "denominator": 0,
90
+ "cascade_correct": 0,
91
+ "only_correct": 0,
92
+ "both_correct": 0,
93
+ "cascade_only_correct": 0,
94
+ "only_only_correct": 0,
95
+ "both_wrong": 0,
96
+ "missing_only": 0,
97
+ "mismatched_examples": 0,
98
+ }
99
+
100
+
101
+ def _finalize_bucket(
102
+ benchmark: str,
103
+ group: str,
104
+ split: str,
105
+ prev_stage_index: int,
106
+ next_stage_index: int,
107
+ prev_model_id: str,
108
+ next_model_id: str,
109
+ only_model_id: str,
110
+ bucket: Dict[str, Any],
111
+ ) -> Dict[str, Any]:
112
+ denom = int(bucket["denominator"])
113
+ cascade_rate = (int(bucket["cascade_correct"]) / denom) if denom else None
114
+ only_rate = (int(bucket["only_correct"]) / denom) if denom else None
115
+ delta = (cascade_rate - only_rate) if (cascade_rate is not None and only_rate is not None) else None
116
+ return {
117
+ "benchmark": benchmark,
118
+ "group": group,
119
+ "split": split,
120
+ "transition": f"stage{prev_stage_index + 1}_wrong_to_stage{next_stage_index + 1}_correct",
121
+ "prev_stage": f"stage{prev_stage_index + 1}",
122
+ "next_stage": f"stage{next_stage_index + 1}",
123
+ "prev_model_id": prev_model_id,
124
+ "next_model_id": next_model_id,
125
+ "only_model_id": only_model_id,
126
+ "denominator": denom,
127
+ "cascade_correct": int(bucket["cascade_correct"]),
128
+ "cascade_rate": cascade_rate,
129
+ "only_correct": int(bucket["only_correct"]),
130
+ "only_rate": only_rate,
131
+ "delta_cascade_minus_only": delta,
132
+ "both_correct": int(bucket["both_correct"]),
133
+ "cascade_only_correct": int(bucket["cascade_only_correct"]),
134
+ "only_only_correct": int(bucket["only_only_correct"]),
135
+ "both_wrong": int(bucket["both_wrong"]),
136
+ "missing_only": int(bucket["missing_only"]),
137
+ "mismatched_examples": int(bucket["mismatched_examples"]),
138
+ }
139
+
140
+
141
+ def _analyze_transition(
142
+ benchmark: str,
143
+ group: str,
144
+ split: str,
145
+ stage_rows: List[Dict[str, Any]],
146
+ only_rows: Dict[int, Dict[str, Any]],
147
+ prev_stage_index: int,
148
+ next_stage_index: int,
149
+ prev_model_id: str,
150
+ next_model_id: str,
151
+ only_model_id: str,
152
+ ) -> Dict[str, Any]:
153
+ bucket = _make_bucket()
154
+ for fallback_index, row in enumerate(stage_rows):
155
+ if _stage_correct(row, prev_stage_index):
156
+ continue
157
+ bucket["denominator"] += 1
158
+ cache_index = _cache_key(row, fallback_index)
159
+ only_row = only_rows.get(cache_index)
160
+ if only_row is None:
161
+ bucket["missing_only"] += 1
162
+ continue
163
+
164
+ if (
165
+ _safe_str(only_row.get("headline")) != _safe_str(row.get("headline"))
166
+ or _safe_str(only_row.get("label")) != _safe_str(row.get("label"))
167
+ ):
168
+ bucket["mismatched_examples"] += 1
169
+
170
+ cascade_ok = _stage_correct(row, next_stage_index)
171
+ only_ok = _only_correct(only_row)
172
+ bucket["cascade_correct"] += int(cascade_ok)
173
+ bucket["only_correct"] += int(only_ok)
174
+ if cascade_ok and only_ok:
175
+ bucket["both_correct"] += 1
176
+ elif cascade_ok:
177
+ bucket["cascade_only_correct"] += 1
178
+ elif only_ok:
179
+ bucket["only_only_correct"] += 1
180
+ else:
181
+ bucket["both_wrong"] += 1
182
+
183
+ return _finalize_bucket(
184
+ benchmark=benchmark,
185
+ group=group,
186
+ split=split,
187
+ prev_stage_index=prev_stage_index,
188
+ next_stage_index=next_stage_index,
189
+ prev_model_id=prev_model_id,
190
+ next_model_id=next_model_id,
191
+ only_model_id=only_model_id,
192
+ bucket=bucket,
193
+ )
194
+
195
+
196
+ def _iter_rows(
197
+ root: Path,
198
+ benchmarks: Sequence[str],
199
+ groups: Sequence[str],
200
+ splits: Sequence[str],
201
+ ) -> Iterable[Dict[str, Any]]:
202
+ for benchmark in benchmarks:
203
+ for group in groups:
204
+ stage_cfgs = MODEL_GROUPS[group]["stages"]
205
+ for split in splits:
206
+ try:
207
+ stage_rows = _load_stage_rows(root, benchmark, group, split)
208
+ except FileNotFoundError:
209
+ continue
210
+ for prev_stage_index, next_stage_index in ((0, 1), (1, 2)):
211
+ only_model_id = _safe_str(stage_cfgs[next_stage_index].get("cache_id"))
212
+ if not only_model_id:
213
+ continue
214
+ try:
215
+ only_rows = _load_only_rows(root, benchmark, only_model_id, split)
216
+ except FileNotFoundError:
217
+ continue
218
+ yield _analyze_transition(
219
+ benchmark=benchmark,
220
+ group=group,
221
+ split=split,
222
+ stage_rows=stage_rows,
223
+ only_rows=only_rows,
224
+ prev_stage_index=prev_stage_index,
225
+ next_stage_index=next_stage_index,
226
+ prev_model_id=_safe_str(stage_cfgs[prev_stage_index].get("cache_id")),
227
+ next_model_id=_safe_str(stage_cfgs[next_stage_index].get("cache_id")),
228
+ only_model_id=only_model_id,
229
+ )
230
+
231
+
232
+ def _fmt_rate(value: Optional[float]) -> str:
233
+ if value is None:
234
+ return "-"
235
+ return f"{100.0 * float(value):.2f}%"
236
+
237
+
238
+ def _fmt_count_rate(count: int, denom: int) -> str:
239
+ if not denom:
240
+ return "0/0"
241
+ return f"{int(count)}/{int(denom)} ({_fmt_rate(count / denom)})"
242
+
243
+
244
+ def _print_rows(rows: Sequence[Dict[str, Any]]) -> None:
245
+ headers = [
246
+ "benchmark",
247
+ "group",
248
+ "split",
249
+ "transition",
250
+ "subset",
251
+ "cascade",
252
+ "only",
253
+ "delta",
254
+ "overlap",
255
+ ]
256
+ printable: List[Dict[str, str]] = []
257
+ for row in rows:
258
+ denom = int(row["denominator"])
259
+ printable.append(
260
+ {
261
+ "benchmark": str(row["benchmark"]),
262
+ "group": str(row["group"]),
263
+ "split": str(row["split"]),
264
+ "transition": f"{row['prev_model_id']} wrong -> {row['next_model_id']} correct",
265
+ "subset": str(denom),
266
+ "cascade": _fmt_count_rate(int(row["cascade_correct"]), denom),
267
+ "only": _fmt_count_rate(int(row["only_correct"]), denom),
268
+ "delta": _fmt_rate(row["delta_cascade_minus_only"]),
269
+ "overlap": (
270
+ f"both={row['both_correct']} "
271
+ f"c_only={row['cascade_only_correct']} "
272
+ f"o_only={row['only_only_correct']} "
273
+ f"none={row['both_wrong']}"
274
+ ),
275
+ }
276
+ )
277
+
278
+ if not printable:
279
+ return
280
+
281
+ widths = {
282
+ header: max(len(header), *(len(item[header]) for item in printable))
283
+ for header in headers
284
+ }
285
+ print(" ".join(header.ljust(widths[header]) for header in headers))
286
+ print(" ".join(("-" * widths[header]) for header in headers))
287
+ for row in printable:
288
+ print(" ".join(row[header].ljust(widths[header]) for header in headers))
289
+
290
+
291
+ def main() -> int:
292
+ parser = argparse.ArgumentParser(description=__doc__)
293
+ parser.add_argument("--root", default="cache/benchmarks", help="Benchmark cache root.")
294
+ parser.add_argument("--benchmarks", nargs="*", help="Benchmarks to include. Default: all.")
295
+ parser.add_argument("--groups", nargs="*", choices=sorted(MODEL_GROUPS), help="Cascade groups to include. Default: all.")
296
+ parser.add_argument("--splits", nargs="*", default=["test"], help="Splits to include. Default: test.")
297
+ parser.add_argument("--output-json", help="Optional path to write JSON results.")
298
+ args = parser.parse_args()
299
+
300
+ root = Path(args.root)
301
+ if not root.exists():
302
+ raise FileNotFoundError(f"Benchmark cache root does not exist: {root}")
303
+
304
+ benchmarks = _iter_benchmarks(root, args.benchmarks)
305
+ groups = list(args.groups) if args.groups else sorted(MODEL_GROUPS)
306
+ splits = [str(split).strip() for split in args.splits if str(split).strip()]
307
+ rows = list(_iter_rows(root, benchmarks, groups, splits))
308
+ rows.sort(key=lambda item: (item["benchmark"], item["group"], item["split"], item["prev_stage"], item["next_stage"]))
309
+
310
+ _print_rows(rows)
311
+
312
+ if args.output_json:
313
+ output_path = Path(args.output_json)
314
+ output_path.parent.mkdir(parents=True, exist_ok=True)
315
+ payload = {
316
+ "root": str(root),
317
+ "benchmarks": benchmarks,
318
+ "groups": groups,
319
+ "splits": splits,
320
+ "rows": rows,
321
+ }
322
+ output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
323
+ return 0
324
+
325
+
326
+ if __name__ == "__main__":
327
+ raise SystemExit(main())
tensorrt.sh ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ TENSORRT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/TensorRT-10.13.0.35
2
+
3
+ grep -q 'export TensorRT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/TensorRT-10.13.0.35' ~/.bashrc || cat >> ~/.bashrc <<EOF
4
+
5
+ # TensorRT
6
+ export TensorRT_ROOT=$TENSORRT_ROOT
7
+ export LD_LIBRARY_PATH=\$TensorRT_ROOT/lib:\${LD_LIBRARY_PATH}
8
+ EOF
9
+
10
+ source ~/.bashrc
11
+
12
+ # 检查
13
+ echo $TensorRT_ROOT
14
+ echo $LD_LIBRARY_PATH