Scripts/launch_all.sh

#!/bin/bash
# Master Launch Script — runs entire pipeline when GPU frees
# Usage: nohup bash Scripts/launch_all.sh > logs/launch_all.log 2>&1 &

source ~/miniconda3/etc/profile.d/conda.sh
conda activate dairygoat
cd ~/goat

echo "=========================================="
echo " FULL PIPELINE LAUNCH"
echo " Start: $(date)"
echo "=========================================="

# Step 0: Wait for GPU
echo "[Step 0] Waiting for GPU > 18GB free..."
while true; do
    FREE=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader | head -1 | awk '{print $1}')
    if [ "$FREE" -gt 18000 ]; then
        echo "  GPU free: ${FREE}MiB — STARTING!"
        break
    fi
    echo "  $(date +%H:%M:%S) GPU free: ${FREE}MiB — waiting..."
    sleep 300
done

# Step 1: CPU Analysis
echo "[Step 1] Running CPU analysis..."
python3 -u Scripts/analyze_cameras.py 2>&1 | tail -20
python3 -u Scripts/analyze_annotations.py 2>&1 | tail -20
echo "  Analysis done."

# Step 2: Cross-Validation
echo "[Step 2] Running cross-validation..."
python3 -u Scripts/eval_crossval.py 2>&1 | tee logs/eval_crossval.log
echo "  Cross-val done."

# Step 3: Checkpoint Ensemble
echo "[Step 3] Running checkpoint ensemble..."
python3 -u Scripts/checkpoint_ensemble.py 2>&1 | tee logs/checkpoint_ensemble.log
echo "  Checkpoint ensemble done."

# Step 4: Adaptive WBF Master Pipeline
echo "[Step 4] Running adaptive WBF pipeline..."
python3 -u Scripts/pipeline_master.py 2>&1 | tee logs/pipeline_master.log
echo "  Adaptive WBF done."

# Step 5: Final Summary
echo "[Step 5] Generating final summary..."
python3 -u -c "
import json, os
results = {}
for name in ['crossval', 'checkpoint_ensemble', 'pipeline_master', 'kitchen_sink_v2']:
    path = f'logs/{name}_results.json'
    if os.path.exists(path):
        with open(path) as f: results[name] = json.load(f)
with open('logs/ALL_RESULTS.json', 'w') as f:
    json.dump(results, f, indent=2)
print('All results saved to logs/ALL_RESULTS.json')
for name, data in sorted(results.items()):
    print(f'  {name}: {json.dumps(data)[:200]}')
"

echo ""
echo "=========================================="
echo " FULL PIPELINE COMPLETE"
echo " End: $(date)"
echo "=========================================="