scripts/test_ucfnet.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT=${ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}
umask "${UCFNET_UMASK:-000}"

usage() {
  cat >&2 <<EOF
Usage: bash scripts/test_ucfnet.sh -w WEIGHTS [-g GPUS] [-c CONFIG] [-m MODE] [-b BATCH] [-s N] [-h]

  -w WEIGHTS  trained model_best.pth                     (required)
  -g GPUS     comma-separated GPU list, e.g. 0,1,2,3     (default: 0)
  -c CONFIG   detector yaml: a bare name under
              DeepfakeBench/training/config/detector/ (e.g. ucfnet.yaml)
              or a path                                  (default: ucfnet.yaml)
  -m MODE     eval mode: cross-domain | in-domain | all  (default: cross-domain)
  -b BATCH    eval batch size                            (default: 96)
  -s N        smoke run on N samples only
  -h          show this help

Flags override the matching env vars (WEIGHTS / GPU_LIST / DETECTOR_TEMPLATE /
EVAL_MODE / BATCH_SIZE / SMOKE_SAMPLES), which still work too.
EOF
}

DETECTOR_DIR="$ROOT/DeepfakeBench/training/config/detector"
while getopts "w:g:c:m:b:s:h" opt; do
  case "$opt" in
    w) WEIGHTS=$OPTARG ;;
    g) GPU_LIST=$OPTARG ;;
    c) if [[ "$OPTARG" == */* ]]; then DETECTOR_TEMPLATE=$OPTARG; else DETECTOR_TEMPLATE="$DETECTOR_DIR/$OPTARG"; fi ;;
    m) EVAL_MODE=$OPTARG ;;
    b) BATCH_SIZE=$OPTARG ;;
    s) SMOKE_SAMPLES=$OPTARG ;;
    h) usage; exit 0 ;;
    *) usage; exit 1 ;;
  esac
done
shift $((OPTIND - 1))

source "$ROOT/scripts/resolve_python.sh"
ENV_PYTHON=$(resolve_env_python "$ROOT")
LIBFIX_DIR=${LIBFIX_DIR:-$ROOT/.libfix}
DATA_ROOT=${DATA_ROOT:-$ROOT/UCF-Net-dataset}
PRETRAINED_ROOT=${PRETRAINED_ROOT:-$ROOT/pretrained}
DETECTOR_TEMPLATE=${DETECTOR_TEMPLATE:-$ROOT/DeepfakeBench/training/config/detector/ucfnet.yaml}
TEST_CONFIG=${TEST_CONFIG:-$ROOT/DeepfakeBench/training/config/test_config.yaml}
RUNTIME_CONFIG=${RUNTIME_CONFIG:-$ROOT/.runtime/configs/ucfnet.yaml}
LOG_DIR=${LOG_DIR:-$ROOT/runs/train/ucfnet}
MODEL_NAME=${MODEL_NAME:-UCF-Net}
DETECTOR_TYPE=${DETECTOR_TYPE:-CLIP-DINO}
EVAL_MODE=${EVAL_MODE:-cross-domain}
GPU_LIST=${GPU_LIST:-0}
BATCH_SIZE=${BATCH_SIZE:-96}
WORKERS=${WORKERS:-4}
PIN_MEMORY=${PIN_MEMORY:-0}
SMOKE_SAMPLES=${SMOKE_SAMPLES:-}
WEIGHTS=${WEIGHTS:?Set WEIGHTS to a trained model_best.pth}

resolve_file() {
  local path=$1
  if [[ "$path" = /* ]]; then
    printf '%s\n' "$path"
  else
    printf '%s\n' "$ROOT/$path"
  fi
}

resolve_dir() {
  local path=$1
  if [[ "$path" = /* ]]; then
    printf '%s\n' "$path"
  else
    printf '%s\n' "$ROOT/$path"
  fi
}

DATA_ROOT=$(resolve_dir "$DATA_ROOT")
PRETRAINED_ROOT=$(resolve_dir "$PRETRAINED_ROOT")
WEIGHTS=$(resolve_file "$WEIGHTS")

case "$EVAL_MODE" in
  cross-domain)
    TEST_JSON=${TEST_JSON:-$DATA_ROOT/splits/test_cross_domain.json}
    EXPECTED_SAMPLES=${EXPECTED_SAMPLES:-286448}
    ;;
  in-domain)
    TEST_JSON=${TEST_JSON:-$DATA_ROOT/splits/test_in_domain.json}
    EXPECTED_SAMPLES=${EXPECTED_SAMPLES:-1393675}
    ;;
  all)
    TEST_JSON=${TEST_JSON:-$DATA_ROOT/splits/test.json}
    EXPECTED_SAMPLES=${EXPECTED_SAMPLES:-1680123}
    ;;
  *)
    echo "Unsupported EVAL_MODE=$EVAL_MODE; use cross-domain, in-domain, or all." >&2
    exit 1
    ;;
esac

TEST_JSON=$(resolve_file "$TEST_JSON")

require_env_python "$ENV_PYTHON"
for path in "$WEIGHTS" "$TEST_JSON" "$TEST_CONFIG"; do
  if [[ ! -f "$path" ]]; then
    echo "Missing required file: $path" >&2
    exit 1
  fi
done

if [[ ! -d "$LIBFIX_DIR" ]]; then
  libcuda_so1=$(ldconfig -p 2>/dev/null | awk '/libcuda\.so\.1/{print $NF; exit}')
  if [[ -n "$libcuda_so1" ]]; then
    mkdir -p "$LIBFIX_DIR"
    ln -sf "$libcuda_so1" "$LIBFIX_DIR/libcuda.so"
  fi
fi
if [[ -d "$LIBFIX_DIR" ]]; then
  export LD_LIBRARY_PATH="$LIBFIX_DIR:/usr/lib/x86_64-linux-gnu${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
fi

"$ENV_PYTHON" "$ROOT/scripts/materialize_config.py" \
  --repo-root "$ROOT" \
  --template "$DETECTOR_TEMPLATE" \
  --output "$RUNTIME_CONFIG" \
  --data-root "$DATA_ROOT" \
  --pretrained-root "$PRETRAINED_ROOT" \
  --log-dir "$LOG_DIR" >/dev/null

IFS=',' read -r -a GPUS <<< "$GPU_LIST"
SHARD_COUNT=${SHARD_COUNT:-${#GPUS[@]}}
RUN_ID=${RUN_ID:-$(basename "$(dirname "$WEIGHTS")")_${EVAL_MODE}_$(date '+%Y-%m-%d-%H-%M-%S')}
OUT_ROOT=${OUT_ROOT:-$ROOT/eval_logs/$RUN_ID}
RESULTS_DIR=${RESULTS_DIR:-$ROOT/results/$RUN_ID}

mkdir -p "$OUT_ROOT/raw" "$OUT_ROOT/logs" "$RESULTS_DIR"
chmod -R a+rwX "$OUT_ROOT" "$RESULTS_DIR" 2>/dev/null || true

export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
export MKL_NUM_THREADS=${MKL_NUM_THREADS:-1}
export OPENBLAS_NUM_THREADS=${OPENBLAS_NUM_THREADS:-1}
export NUMEXPR_NUM_THREADS=${NUMEXPR_NUM_THREADS:-1}
export TORCH_NUM_THREADS=${TORCH_NUM_THREADS:-1}
export PIN_MEMORY

{
  echo "started_at=$(date '+%F %T')"
  echo "root=$ROOT"
  echo "env_python=$ENV_PYTHON"
  echo "eval_mode=$EVAL_MODE"
  echo "test_json=$TEST_JSON"
  echo "dataset_root=$DATA_ROOT"
  echo "detector=$RUNTIME_CONFIG"
  echo "weights=$WEIGHTS"
  echo "gpu_list=$GPU_LIST"
  echo "shard_count=$SHARD_COUNT"
  echo "batch_size=$BATCH_SIZE"
  echo "workers=$WORKERS"
  echo "expected_samples=$EXPECTED_SAMPLES"
  echo "smoke_samples=$SMOKE_SAMPLES"
  echo "out_root=$OUT_ROOT"
  echo "results_dir=$RESULTS_DIR"
  if command -v nvidia-smi >/dev/null 2>&1; then
    nvidia-smi --query-gpu=index,name,memory.total,memory.used,utilization.gpu --format=csv,noheader,nounits
  fi
} | tee "$OUT_ROOT/logs/run_manifest.txt"

cat > "$OUT_ROOT/run_command.sh" <<EOF
#!/usr/bin/env bash
cd "$ROOT"
WEIGHTS="$WEIGHTS" EVAL_MODE="$EVAL_MODE" GPU_LIST="$GPU_LIST" BATCH_SIZE="$BATCH_SIZE" WORKERS="$WORKERS" SHARD_COUNT="$SHARD_COUNT" OUT_ROOT="$OUT_ROOT" RESULTS_DIR="$RESULTS_DIR" bash scripts/test_ucfnet.sh
EOF
chmod +x "$OUT_ROOT/run_command.sh"

pids=()
for shard in $(seq 0 $((SHARD_COUNT - 1))); do
  gpu=${GPUS[$((shard % ${#GPUS[@]}))]}
  extra_args=()
  if [[ -n "$SMOKE_SAMPLES" ]]; then
    extra_args+=(--max-samples "$SMOKE_SAMPLES")
  fi
  echo "[$(date '+%F %T')] starting $MODEL_NAME $EVAL_MODE shard $shard/$SHARD_COUNT on cuda:$gpu" | tee -a "$OUT_ROOT/logs/run.log"
  CUDA_VISIBLE_DEVICES="$gpu" "$ENV_PYTHON" "$ROOT/scripts/eval_test_json.py" \
    --repo-root "$ROOT" \
    --workdir "$ROOT/DeepfakeBench" \
    --detector-path "$RUNTIME_CONFIG" \
    --test-config-path "$TEST_CONFIG" \
    --weights-path "$WEIGHTS" \
    --test-json "$TEST_JSON" \
    --dataset-root "$DATA_ROOT" \
    --batch-size "$BATCH_SIZE" \
    --workers "$WORKERS" \
    --shard-index "$shard" \
    --shard-count "$SHARD_COUNT" \
    --raw-output "$OUT_ROOT/raw/shard_${shard}.npz" \
    --log-file "$OUT_ROOT/logs/shard_${shard}.log" \
    "${extra_args[@]}" \
    > "$OUT_ROOT/logs/shard_${shard}.stdout" 2>&1 &
  pids+=("$!")
  sleep 5
done

failed=0
for pid in "${pids[@]}"; do
  if ! wait "$pid"; then
    failed=1
  fi
done

if [[ "$failed" != 0 ]]; then
  echo "[$(date '+%F %T')] $MODEL_NAME $EVAL_MODE evaluation failed; inspect $OUT_ROOT/logs" | tee -a "$OUT_ROOT/logs/run.log" >&2
  exit 1
fi

merge_args=(
  "$ROOT/scripts/merge_ucfnet_results.py"
  --model-raw "$OUT_ROOT"/raw/shard_*.npz
  --output-dir "$OUT_ROOT"
  --results-dir "$RESULTS_DIR"
  --model-name "$MODEL_NAME"
  --detector-type "$DETECTOR_TYPE"
  --eval-mode "$EVAL_MODE"
)
if [[ -n "$SMOKE_SAMPLES" ]]; then
  merge_args+=(--skip-sample-check --allow-partial-metrics)
else
  merge_args+=(--expected-samples "$EXPECTED_SAMPLES")
fi

"$ENV_PYTHON" "${merge_args[@]}" | tee -a "$OUT_ROOT/logs/run.log"
echo "[$(date '+%F %T')] wrote compact results to $RESULTS_DIR" | tee -a "$OUT_ROOT/logs/run.log"