| #!/bin/bash |
| |
| |
| |
| |
| |
| set -uo pipefail |
| D=/home/haojinw2 |
| OUT=$D/_pack_data |
| rm -rf "$OUT"; mkdir -p "$OUT" |
|
|
| echo "=== evalout: 按目录分组打包(每组约 15 个目录)" |
| cd $D/evalout |
| ls -d */ 2>/dev/null | sed 's#/##' | sort > /tmp/_dirs.txt |
| total=$(wc -l < /tmp/_dirs.txt) |
| echo " 共 $total 个目录" |
| n=0; g=0 |
| split -l 15 /tmp/_dirs.txt /tmp/_grp_ |
| for grp in /tmp/_grp_*; do |
| g=$((g+1)) |
| name=$(printf "evalout_%02d.tar.gz" $g) |
| tar -czf "$OUT/$name" -T "$grp" 2>/dev/null |
| sz=$(du -m "$OUT/$name" | cut -f1) |
| cnt=$(wc -l < "$grp") |
| printf " %-22s %4sM (%s 个目录)\n" "$name" "$sz" "$cnt" |
| done |
| rm -f /tmp/_grp_* /tmp/_dirs.txt |
| |
| cd $D/evalout && find . -maxdepth 1 -type f | head -1 >/dev/null 2>&1 && \ |
| tar -czf "$OUT/evalout_loose.tar.gz" $(find . -maxdepth 1 -type f -printf "%P\n" 2>/dev/null) 2>/dev/null && \ |
| echo " evalout_loose.tar.gz $(du -m $OUT/evalout_loose.tar.gz | cut -f1)M" |
|
|
| echo "=== 其它目录" |
| cd $D |
| for x in datasets iter_data_keep; do |
| [ -d "$x" ] || continue |
| tar -czf "$OUT/${x}.tar.gz" "$x" 2>/dev/null |
| printf " %-22s %4sM\n" "${x}.tar.gz" "$(du -m $OUT/${x}.tar.gz | cut -f1)" |
| done |
| tar -czf "$OUT/sciworld_sft.tar.gz" sciworld_sft_teacher32b_*.jsonl 2>/dev/null |
| printf " %-22s %4sM\n" "sciworld_sft.tar.gz" "$(du -m $OUT/sciworld_sft.tar.gz | cut -f1)" |
|
|
| echo |
| echo "=== 汇总" |
| before=$(du -sm $D/evalout $D/datasets $D/iter_data_keep $D/sciworld_sft_teacher32b_*.jsonl 2>/dev/null | awk '{s+=$1} END {print s}') |
| after=$(du -sm "$OUT" | cut -f1) |
| echo " 原始 $((before/1024))G -> 打包后 $((after/1024))G (压缩 $(awk -v a=$before -v b=$after 'BEGIN{printf "%.1f", a/b}')x)" |
| echo " 文件数 1462 -> $(ls "$OUT" | wc -l)" |
| df -h /home | tail -1 | awk '{print " 磁盘 "$4" 可用"}' |
|
|
