#!/bin/bash
# 把 data-dq 的内容打包: 1462 个文件 -> 约 20 个 tar.gz
#
# 两个原因必须打包:
#   1. HF 限制每仓库每小时 256 次 commit, 而 upload_file 一个文件一次 commit
#   2. rollout 是纯文本 jsonl, gzip -1 实测 5.2x(261M -> 50M, 2 秒)
set -uo pipefail
D=/home/haojinw2
OUT=$D/_pack_data
rm -rf "$OUT"; mkdir -p "$OUT"

echo "=== evalout: 按目录分组打包(每组约 15 个目录)"
cd $D/evalout
ls -d */ 2>/dev/null | sed 's#/##' | sort > /tmp/_dirs.txt
total=$(wc -l < /tmp/_dirs.txt)
echo "  共 $total 个目录"
n=0; g=0
split -l 15 /tmp/_dirs.txt /tmp/_grp_
for grp in /tmp/_grp_*; do
  g=$((g+1))
  name=$(printf "evalout_%02d.tar.gz" $g)
  tar -czf "$OUT/$name" -T "$grp" 2>/dev/null
  sz=$(du -m "$OUT/$name" | cut -f1)
  cnt=$(wc -l < "$grp")
  printf "  %-22s %4sM  (%s 个目录)\n" "$name" "$sz" "$cnt"
done
rm -f /tmp/_grp_* /tmp/_dirs.txt
# evalout 根下的散文件
cd $D/evalout && find . -maxdepth 1 -type f | head -1 >/dev/null 2>&1 && \
  tar -czf "$OUT/evalout_loose.tar.gz" $(find . -maxdepth 1 -type f -printf "%P\n" 2>/dev/null) 2>/dev/null && \
  echo "  evalout_loose.tar.gz  $(du -m $OUT/evalout_loose.tar.gz | cut -f1)M"

echo "=== 其它目录"
cd $D
for x in datasets iter_data_keep; do
  [ -d "$x" ] || continue
  tar -czf "$OUT/${x}.tar.gz" "$x" 2>/dev/null
  printf "  %-22s %4sM\n" "${x}.tar.gz" "$(du -m $OUT/${x}.tar.gz | cut -f1)"
done
tar -czf "$OUT/sciworld_sft.tar.gz" sciworld_sft_teacher32b_*.jsonl 2>/dev/null
printf "  %-22s %4sM\n" "sciworld_sft.tar.gz" "$(du -m $OUT/sciworld_sft.tar.gz | cut -f1)"

echo
echo "=== 汇总"
before=$(du -sm $D/evalout $D/datasets $D/iter_data_keep $D/sciworld_sft_teacher32b_*.jsonl 2>/dev/null | awk '{s+=$1} END {print s}')
after=$(du -sm "$OUT" | cut -f1)
echo "  原始 $((before/1024))G -> 打包后 $((after/1024))G  (压缩 $(awk -v a=$before -v b=$after 'BEGIN{printf "%.1f", a/b}')x)"
echo "  文件数 1462 -> $(ls "$OUT" | wc -l)"
df -h /home | tail -1 | awk '{print "  磁盘 "$4" 可用"}'