script / fast_rm.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
734d13f verified
Raw History Blame Contribute Delete
8.08 kB
#!/usr/bin/env bash
set -euo pipefail
usage() {
echo "Usage: bash $0 TARGET_DIR [JOBS]" >&2
exit 2
}
die() {
echo "fast_rm: $*" >&2
exit 2
}
format_bytes() {
awk -v bytes="${1:-0}" 'BEGIN {
split("B KiB MiB GiB TiB PiB", units, " ")
value = bytes + 0
unit = 1
while (value >= 1024 && unit < 6) {
value /= 1024
unit++
}
if (unit == 1) {
printf "%.0f %s", value, units[unit]
} else {
printf "%.2f %s", value, units[unit]
}
}'
}
fs_avail_bytes() {
df -B1 --output=avail "$1" 2>/dev/null | awk 'NR==2 {print $1}'
}
progress_bar() {
local freed=$1
local total=$2
local items=$3
local final=${4:-0}
local now elapsed percent filled empty bar rate capped
local width=30
now=$(date +%s)
elapsed=$((now - start_ts))
[ "$elapsed" -gt 0 ] || elapsed=1
[ "$freed" -ge 0 ] || freed=0
rate=$(awk -v b="$freed" -v s="$elapsed" 'BEGIN { if (s <= 0) s = 1; printf "%.0f", b / s }')
if [ "$total" -gt 0 ]; then
capped=$freed
[ "$capped" -le "$total" ] || capped=$total
percent=$((capped * 100 / total))
[ "$percent" -le 100 ] || percent=100
else
percent=0
fi
if [ "$total" -gt 0 ]; then
filled=$((percent * width / 100))
empty=$((width - filled))
bar=$(printf '%*s' "$filled" '' | tr ' ' '#')
bar=$bar$(printf '%*s' "$empty" '' | tr ' ' '-')
printf '\rfast_rm: [%s] %3d%% freed %s/%s rate %s/s items %d' \
"$bar" "$percent" "$(format_bytes "$freed")" "$(format_bytes "$total")" "$(format_bytes "$rate")" "$items" >&2
else
printf '\rfast_rm: freed %s rate %s/s items %d' \
"$(format_bytes "$freed")" "$(format_bytes "$rate")" "$items" >&2
fi
if [ "$final" = 1 ]; then
printf '\n' >&2
fi
}
monitor_progress() {
local total=$1
local watch_path=$2
local start_avail=$3
local current_avail=0
local freed=0
local max_freed=0
local items=0
local last_render=0
local now msg
while true; do
if IFS= read -r -t "$progress_interval" msg; then
[ "$msg" != "__FAST_RM_DONE__" ] || break
items=$((items + 1))
fi
current_avail=$(fs_avail_bytes "$watch_path")
case "$current_avail" in
''|*[!0-9]*) current_avail=$start_avail ;;
esac
freed=$((current_avail - start_avail))
[ "$freed" -ge 0 ] || freed=0
if [ "$freed" -gt "$max_freed" ]; then
max_freed=$freed
fi
now=$(date +%s)
if [ $((now - last_render)) -ge "$progress_interval" ]; then
progress_bar "$max_freed" "$total" "$items" 0
last_render=$now
fi
done
current_avail=$(fs_avail_bytes "$watch_path")
case "$current_avail" in
''|*[!0-9]*) current_avail=$start_avail ;;
esac
freed=$((current_avail - start_avail))
[ "$freed" -ge 0 ] || freed=0
if [ "$freed" -gt "$max_freed" ]; then
max_freed=$freed
fi
progress_bar "$max_freed" "$total" "$items" 1
}
delete_stage() {
local min_depth=$1
local max_depth=$2
if [ "$progress" = 1 ]; then
find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \
| FAST_RM_PROGRESS_FIFO=$progress_fifo xargs -0 -r -n 1 -P "$jobs" bash -c '
set -e
for path do
rm -rf -- "$path"
printf "1\n" > "$FAST_RM_PROGRESS_FIFO" || true
done
' _
else
find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \
| xargs -0 -r -n 1 -P "$jobs" rm -rf --
fi
}
[ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
target=$1
jobs=${2:-${FAST_RM_JOBS:-$(nproc 2>/dev/null || echo 32)}}
shard_depth=${FAST_RM_SHARD_DEPTH:-3}
progress=${FAST_RM_PROGRESS:-1}
progress_interval=${FAST_RM_PROGRESS_INTERVAL:-1}
exact_total=${FAST_RM_EXACT_TOTAL:-0}
case "$jobs" in
''|*[!0-9]*) die "JOBS must be a positive integer" ;;
esac
case "$shard_depth" in
''|*[!0-9]*) die "FAST_RM_SHARD_DEPTH must be a positive integer" ;;
esac
case "$progress" in
0|1) ;;
*) die "FAST_RM_PROGRESS must be 0 or 1" ;;
esac
case "$exact_total" in
0|1) ;;
*) die "FAST_RM_EXACT_TOTAL must be 0 or 1" ;;
esac
case "$progress_interval" in
''|*[!0-9]*) die "FAST_RM_PROGRESS_INTERVAL must be a positive integer" ;;
esac
[ "$jobs" -gt 0 ] || die "JOBS must be > 0"
[ "$shard_depth" -gt 0 ] || die "FAST_RM_SHARD_DEPTH must be > 0"
[ "$progress_interval" -gt 0 ] || die "FAST_RM_PROGRESS_INTERVAL must be > 0"
while [ "$target" != "/" ] && [ "${target%/}" != "$target" ]; do
target=${target%/}
done
[ -n "$target" ] || die "empty target"
[ "$target" != "/" ] || die "refuse to delete /"
parent=$(dirname -- "$target")
base=$(basename -- "$target")
[ "$base" != "." ] && [ "$base" != ".." ] || die "bad target: $target"
parent_abs=$(realpath -- "$parent")
target_abs=$parent_abs/$base
if [ ! -e "$target_abs" ] && [ ! -L "$target_abs" ]; then
echo "fast_rm: not found: $target_abs"
exit 0
fi
home_abs=$(realpath -- "${HOME:-/nonexistent}" 2>/dev/null || true)
case "$target_abs" in
"/"|"/cpfs01"|"/cpfs01/users"|"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong"|"/mnt"|"/mnt/workspace")
die "refuse to delete protected path: $target_abs"
;;
esac
[ -z "$home_abs" ] || [ "$target_abs" != "$home_abs" ] || die "refuse to delete HOME: $target_abs"
mount_point=$(findmnt -T "$target_abs" -n -o TARGET 2>/dev/null || true)
[ -z "$mount_point" ] || [ "$mount_point" != "$target_abs" ] || die "refuse to delete mount point: $target_abs"
cwd=$(pwd -P)
case "$cwd/" in
"$target_abs"/*|"$target_abs/"*) die "refuse to delete current working tree: $target_abs" ;;
esac
if [ ! -d "$target_abs" ] || [ -L "$target_abs" ]; then
rm -f -- "$target_abs"
exit 0
fi
progress_fifo=
progress_pid=
progress_keepalive_fd=
cleanup_progress() {
if [ -n "$progress_pid" ]; then
kill "$progress_pid" 2>/dev/null || true
fi
if [ -n "$progress_keepalive_fd" ]; then
exec {progress_keepalive_fd}>&- 2>/dev/null || true
fi
if [ -n "$progress_fifo" ]; then
rm -f -- "$progress_fifo"
fi
}
trap cleanup_progress EXIT
safe_base=$(printf '%s' "$base" | tr -c 'A-Za-z0-9._-' '_')
if [[ "$base" == .deleting_* ]]; then
trash=$target_abs
else
trash=$parent_abs/.deleting_${safe_base}.$(date +%Y%m%d_%H%M%S).$$
mv -- "$target_abs" "$trash"
fi
echo "fast_rm: deleting $trash with jobs=$jobs shard_depth=$shard_depth"
if [ "$progress" = 1 ]; then
total_bytes=0
if [ "$exact_total" = 1 ]; then
echo "fast_rm: exact sizing target..." >&2
total_bytes=$(du -sB1 -- "$trash" 2>/dev/null | awk 'NR==1 {print $1}')
case "$total_bytes" in
''|*[!0-9]*) total_bytes=0 ;;
esac
echo "fast_rm: total $(format_bytes "$total_bytes")" >&2
else
echo "fast_rm: skip exact sizing; showing freed space estimate" >&2
fi
start_avail=$(fs_avail_bytes "$parent_abs")
case "$start_avail" in
''|*[!0-9]*) start_avail=0 ;;
esac
progress_fifo=$(mktemp -u "${TMPDIR:-/tmp}/fast_rm_progress.XXXXXX")
mkfifo "$progress_fifo"
start_ts=$(date +%s)
monitor_progress "$total_bytes" "$parent_abs" "$start_avail" < "$progress_fifo" &
progress_pid=$!
exec {progress_keepalive_fd}>"$progress_fifo"
fi
if [ "$shard_depth" -gt 1 ]; then
delete_stage "$shard_depth" "$shard_depth"
fi
delete_stage 1 1
rmdir -- "$trash" 2>/dev/null || rm -rf -- "$trash"
if [ "$progress" = 1 ]; then
printf '%s\n' "__FAST_RM_DONE__" >&$progress_keepalive_fd || true
wait "$progress_pid" 2>/dev/null || true
progress_pid=
exec {progress_keepalive_fd}>&- 2>/dev/null || true
progress_keepalive_fd=
rm -f -- "$progress_fifo"
progress_fifo=
fi
echo "fast_rm: done"