Download fast_rm.sh from Tinghong-Ye22/script: direct link, hf CLI and curl.
- Browser
- Download file 8.08 kB
-
https://huggingface.co/Tinghong-Ye22/script/resolve/main/fast_rm.sh
- Command line
-
hf download hf://Tinghong-Ye22/script/fast_rm.sh
-
curl -L -o fast_rm.sh https://huggingface.co/Tinghong-Ye22/script/resolve/main/fast_rm.sh
8.08 kB
| set -euo pipefail | |
| usage() { | |
| echo "Usage: bash $0 TARGET_DIR [JOBS]" >&2 | |
| exit 2 | |
| } | |
| die() { | |
| echo "fast_rm: $*" >&2 | |
| exit 2 | |
| } | |
| format_bytes() { | |
| awk -v bytes="${1:-0}" 'BEGIN { | |
| split("B KiB MiB GiB TiB PiB", units, " ") | |
| value = bytes + 0 | |
| unit = 1 | |
| while (value >= 1024 && unit < 6) { | |
| value /= 1024 | |
| unit++ | |
| } | |
| if (unit == 1) { | |
| printf "%.0f %s", value, units[unit] | |
| } else { | |
| printf "%.2f %s", value, units[unit] | |
| } | |
| }' | |
| } | |
| fs_avail_bytes() { | |
| df -B1 --output=avail "$1" 2>/dev/null | awk 'NR==2 {print $1}' | |
| } | |
| progress_bar() { | |
| local freed=$1 | |
| local total=$2 | |
| local items=$3 | |
| local final=${4:-0} | |
| local now elapsed percent filled empty bar rate capped | |
| local width=30 | |
| now=$(date +%s) | |
| elapsed=$((now - start_ts)) | |
| [ "$elapsed" -gt 0 ] || elapsed=1 | |
| [ "$freed" -ge 0 ] || freed=0 | |
| rate=$(awk -v b="$freed" -v s="$elapsed" 'BEGIN { if (s <= 0) s = 1; printf "%.0f", b / s }') | |
| if [ "$total" -gt 0 ]; then | |
| capped=$freed | |
| [ "$capped" -le "$total" ] || capped=$total | |
| percent=$((capped * 100 / total)) | |
| [ "$percent" -le 100 ] || percent=100 | |
| else | |
| percent=0 | |
| fi | |
| if [ "$total" -gt 0 ]; then | |
| filled=$((percent * width / 100)) | |
| empty=$((width - filled)) | |
| bar=$(printf '%*s' "$filled" '' | tr ' ' '#') | |
| bar=$bar$(printf '%*s' "$empty" '' | tr ' ' '-') | |
| printf '\rfast_rm: [%s] %3d%% freed %s/%s rate %s/s items %d' \ | |
| "$bar" "$percent" "$(format_bytes "$freed")" "$(format_bytes "$total")" "$(format_bytes "$rate")" "$items" >&2 | |
| else | |
| printf '\rfast_rm: freed %s rate %s/s items %d' \ | |
| "$(format_bytes "$freed")" "$(format_bytes "$rate")" "$items" >&2 | |
| fi | |
| if [ "$final" = 1 ]; then | |
| printf '\n' >&2 | |
| fi | |
| } | |
| monitor_progress() { | |
| local total=$1 | |
| local watch_path=$2 | |
| local start_avail=$3 | |
| local current_avail=0 | |
| local freed=0 | |
| local max_freed=0 | |
| local items=0 | |
| local last_render=0 | |
| local now msg | |
| while true; do | |
| if IFS= read -r -t "$progress_interval" msg; then | |
| [ "$msg" != "__FAST_RM_DONE__" ] || break | |
| items=$((items + 1)) | |
| fi | |
| current_avail=$(fs_avail_bytes "$watch_path") | |
| case "$current_avail" in | |
| ''|*[!0-9]*) current_avail=$start_avail ;; | |
| esac | |
| freed=$((current_avail - start_avail)) | |
| [ "$freed" -ge 0 ] || freed=0 | |
| if [ "$freed" -gt "$max_freed" ]; then | |
| max_freed=$freed | |
| fi | |
| now=$(date +%s) | |
| if [ $((now - last_render)) -ge "$progress_interval" ]; then | |
| progress_bar "$max_freed" "$total" "$items" 0 | |
| last_render=$now | |
| fi | |
| done | |
| current_avail=$(fs_avail_bytes "$watch_path") | |
| case "$current_avail" in | |
| ''|*[!0-9]*) current_avail=$start_avail ;; | |
| esac | |
| freed=$((current_avail - start_avail)) | |
| [ "$freed" -ge 0 ] || freed=0 | |
| if [ "$freed" -gt "$max_freed" ]; then | |
| max_freed=$freed | |
| fi | |
| progress_bar "$max_freed" "$total" "$items" 1 | |
| } | |
| delete_stage() { | |
| local min_depth=$1 | |
| local max_depth=$2 | |
| if [ "$progress" = 1 ]; then | |
| find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \ | |
| | FAST_RM_PROGRESS_FIFO=$progress_fifo xargs -0 -r -n 1 -P "$jobs" bash -c ' | |
| set -e | |
| for path do | |
| rm -rf -- "$path" | |
| printf "1\n" > "$FAST_RM_PROGRESS_FIFO" || true | |
| done | |
| ' _ | |
| else | |
| find "$trash" -mindepth "$min_depth" -maxdepth "$max_depth" -print0 \ | |
| | xargs -0 -r -n 1 -P "$jobs" rm -rf -- | |
| fi | |
| } | |
| [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage | |
| target=$1 | |
| jobs=${2:-${FAST_RM_JOBS:-$(nproc 2>/dev/null || echo 32)}} | |
| shard_depth=${FAST_RM_SHARD_DEPTH:-3} | |
| progress=${FAST_RM_PROGRESS:-1} | |
| progress_interval=${FAST_RM_PROGRESS_INTERVAL:-1} | |
| exact_total=${FAST_RM_EXACT_TOTAL:-0} | |
| case "$jobs" in | |
| ''|*[!0-9]*) die "JOBS must be a positive integer" ;; | |
| esac | |
| case "$shard_depth" in | |
| ''|*[!0-9]*) die "FAST_RM_SHARD_DEPTH must be a positive integer" ;; | |
| esac | |
| case "$progress" in | |
| 0|1) ;; | |
| *) die "FAST_RM_PROGRESS must be 0 or 1" ;; | |
| esac | |
| case "$exact_total" in | |
| 0|1) ;; | |
| *) die "FAST_RM_EXACT_TOTAL must be 0 or 1" ;; | |
| esac | |
| case "$progress_interval" in | |
| ''|*[!0-9]*) die "FAST_RM_PROGRESS_INTERVAL must be a positive integer" ;; | |
| esac | |
| [ "$jobs" -gt 0 ] || die "JOBS must be > 0" | |
| [ "$shard_depth" -gt 0 ] || die "FAST_RM_SHARD_DEPTH must be > 0" | |
| [ "$progress_interval" -gt 0 ] || die "FAST_RM_PROGRESS_INTERVAL must be > 0" | |
| while [ "$target" != "/" ] && [ "${target%/}" != "$target" ]; do | |
| target=${target%/} | |
| done | |
| [ -n "$target" ] || die "empty target" | |
| [ "$target" != "/" ] || die "refuse to delete /" | |
| parent=$(dirname -- "$target") | |
| base=$(basename -- "$target") | |
| [ "$base" != "." ] && [ "$base" != ".." ] || die "bad target: $target" | |
| parent_abs=$(realpath -- "$parent") | |
| target_abs=$parent_abs/$base | |
| if [ ! -e "$target_abs" ] && [ ! -L "$target_abs" ]; then | |
| echo "fast_rm: not found: $target_abs" | |
| exit 0 | |
| fi | |
| home_abs=$(realpath -- "${HOME:-/nonexistent}" 2>/dev/null || true) | |
| case "$target_abs" in | |
| "/"|"/cpfs01"|"/cpfs01/users"|"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong"|"/mnt"|"/mnt/workspace") | |
| die "refuse to delete protected path: $target_abs" | |
| ;; | |
| esac | |
| [ -z "$home_abs" ] || [ "$target_abs" != "$home_abs" ] || die "refuse to delete HOME: $target_abs" | |
| mount_point=$(findmnt -T "$target_abs" -n -o TARGET 2>/dev/null || true) | |
| [ -z "$mount_point" ] || [ "$mount_point" != "$target_abs" ] || die "refuse to delete mount point: $target_abs" | |
| cwd=$(pwd -P) | |
| case "$cwd/" in | |
| "$target_abs"/*|"$target_abs/"*) die "refuse to delete current working tree: $target_abs" ;; | |
| esac | |
| if [ ! -d "$target_abs" ] || [ -L "$target_abs" ]; then | |
| rm -f -- "$target_abs" | |
| exit 0 | |
| fi | |
| progress_fifo= | |
| progress_pid= | |
| progress_keepalive_fd= | |
| cleanup_progress() { | |
| if [ -n "$progress_pid" ]; then | |
| kill "$progress_pid" 2>/dev/null || true | |
| fi | |
| if [ -n "$progress_keepalive_fd" ]; then | |
| exec {progress_keepalive_fd}>&- 2>/dev/null || true | |
| fi | |
| if [ -n "$progress_fifo" ]; then | |
| rm -f -- "$progress_fifo" | |
| fi | |
| } | |
| trap cleanup_progress EXIT | |
| safe_base=$(printf '%s' "$base" | tr -c 'A-Za-z0-9._-' '_') | |
| if [[ "$base" == .deleting_* ]]; then | |
| trash=$target_abs | |
| else | |
| trash=$parent_abs/.deleting_${safe_base}.$(date +%Y%m%d_%H%M%S).$$ | |
| mv -- "$target_abs" "$trash" | |
| fi | |
| echo "fast_rm: deleting $trash with jobs=$jobs shard_depth=$shard_depth" | |
| if [ "$progress" = 1 ]; then | |
| total_bytes=0 | |
| if [ "$exact_total" = 1 ]; then | |
| echo "fast_rm: exact sizing target..." >&2 | |
| total_bytes=$(du -sB1 -- "$trash" 2>/dev/null | awk 'NR==1 {print $1}') | |
| case "$total_bytes" in | |
| ''|*[!0-9]*) total_bytes=0 ;; | |
| esac | |
| echo "fast_rm: total $(format_bytes "$total_bytes")" >&2 | |
| else | |
| echo "fast_rm: skip exact sizing; showing freed space estimate" >&2 | |
| fi | |
| start_avail=$(fs_avail_bytes "$parent_abs") | |
| case "$start_avail" in | |
| ''|*[!0-9]*) start_avail=0 ;; | |
| esac | |
| progress_fifo=$(mktemp -u "${TMPDIR:-/tmp}/fast_rm_progress.XXXXXX") | |
| mkfifo "$progress_fifo" | |
| start_ts=$(date +%s) | |
| monitor_progress "$total_bytes" "$parent_abs" "$start_avail" < "$progress_fifo" & | |
| progress_pid=$! | |
| exec {progress_keepalive_fd}>"$progress_fifo" | |
| fi | |
| if [ "$shard_depth" -gt 1 ]; then | |
| delete_stage "$shard_depth" "$shard_depth" | |
| fi | |
| delete_stage 1 1 | |
| rmdir -- "$trash" 2>/dev/null || rm -rf -- "$trash" | |
| if [ "$progress" = 1 ]; then | |
| printf '%s\n' "__FAST_RM_DONE__" >&$progress_keepalive_fd || true | |
| wait "$progress_pid" 2>/dev/null || true | |
| progress_pid= | |
| exec {progress_keepalive_fd}>&- 2>/dev/null || true | |
| progress_keepalive_fd= | |
| rm -f -- "$progress_fifo" | |
| progress_fifo= | |
| fi | |
| echo "fast_rm: done" | |