File size: 13,237 Bytes
1f52672 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | #!/usr/bin/env bash
# ==============================================================================
# 01_prepare_data.sh – Stage 1: Synthetic Data Forge, Model Verification & Unsloth Preparation
# ==============================================================================
# ⚠️ SPECIALIZED PIPELINE NOTICE:
# This script is exclusively engineered for preparing data and downloading checkpoints for the Indian Digital Personal
# Data Protection (DPDP) Act 2023 & Rules 2025 specialized legal models on NVIDIA DGX Spark infrastructure (128 GB VRAM).
# It checks/downloads our Teacher Model (`Qwen2-72B-Instruct-FP8`) and Student Model (`Qwen2.5-7B-Instruct`), synthesizes
# statutory training data, and formats Unsloth SFT + DPO datasets. It is specialized for our DPDP law pipeline only.
#
# Prerequisites for DGX Spark Execution:
# - NVIDIA DGX / GPU environment (128 GB VRAM unified or split)
# - Python 3.10+ / 3.12 with CUDA 12.x/13.x support
# - Packages installed (`pip install -r ml/requirements.txt`): unsloth, trl, transformers, vllm, datasets, torch, huggingface_hub, chromadb, langchain-text-splitters, rank_bm25
#
# Usage:
# bash ml/scripts/01_prepare_data.sh [--skip-gan]
#
# Options:
# --skip-gan Skip running gan_forge.py (useful if raw GAN policy files exist)
# -h, --help Show help menu and exit
# ==============================================================================
set -euo pipefail
# Locate repository and ML root relative to this script
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ML_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
REPO_ROOT="$(cd "${ML_DIR}/.." && pwd)"
# Resolve Python binary
PYTHON_CMD="python"
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
fi
SKIP_GAN=false
while [[ $# -gt 0 ]]; do
case $1 in
--skip-gan)
SKIP_GAN=true
shift
;;
-h|--help)
echo "Usage: bash ml/scripts/01_prepare_data.sh [options]"
echo "Options:"
echo " --skip-gan Skip running gan_forge.py and proceed directly to tree building and dataset formatting"
echo " -h, --help Show this help message and exit"
exit 0
;;
*)
echo "❌ Unknown option: $1"
echo "Use --help for available options."
exit 1
;;
esac
done
# Setup logging
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
export LOG_DIR="${REPO_ROOT}/logs"
export PYTHONUNBUFFERED=1
mkdir -p "${LOG_DIR}"
SCRIPT_NAME=$(basename "$0" .sh)
MASTER_LOG="${LOG_DIR}/${SCRIPT_NAME}_${TIMESTAMP}.log"
exec > >(tee -i "${MASTER_LOG}")
exec 2>&1
echo "════════════════════════════════════════════════════════════════════════════════"
echo "🚀 INITIATING STAGE 1: DGX SPARK MODEL VERIFICATION & DATA PREPARATION"
echo "════════════════════════════════════════════════════════════════════════════════"
echo " • Repository Root: ${REPO_ROOT}"
echo " • ML Root: ${ML_DIR}"
echo " • Python Binary: $(command -v "${PYTHON_CMD}")"
echo " • Logs Directory: ${LOG_DIR}"
echo " • Skip GAN Forge: ${SKIP_GAN}"
echo " • Target Domain: Digital Personal Data Protection (DPDP) Act 2023 & Rules 2025"
echo "════════════════════════════════════════════════════════════════════════════════"
# Terminate lingering vLLM API server processes from aborted jobs if any
echo -e "\n🧹 Cleaning up any orphaned vLLM API endpoints before starting..."
pkill -f vllm.entrypoints.openai.api_server 2>/dev/null || true
# ════════════════════════════════════════════════════════════════════════════════
# [PHASE 0.1]: PURGE CACHES (PRESERVING GENERATED DATA)
# ════════════════════════════════════════════════════════════════════════════════
echo -e "\n🧹 [PHASE 0.1]: Purging Caches..."
# 1. Purge Python Bytecode Cache
find "${REPO_ROOT}" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
echo " ✅ Purged all __pycache__ directories."
# 2. Purge Vector DB Cache
rm -f "${ML_DIR}/data-forge/dpdp_hybrid_index.pkl" 2>/dev/null || true
echo " ✅ Purged Hybrid RAG Index cache."
# 3. Purge Aggregated JSONL data
rm -f "${ML_DIR}/slm-training/data"/*.jsonl 2>/dev/null || true
echo " ✅ Purged aggregated JSONL training files (Will be rebuilt in Phase 3)."
# ════════════════════════════════════════════════════════════════════════════════
# [PHASE 0.5/4]: MODEL VERIFICATION & DOWNLOAD (LOCAL CHECK & HUGGINGFACE FALLBACK)
# ════════════════════════════════════════════════════════════════════════════════
mkdir -p "${ML_DIR}/models"
echo -e "\n▶️ [PHASE 0.5/4]: Verifying Teacher ('Qwen2-72B') & Student ('Qwen2.5-7B-Instruct') Models in ml/models/..."
# 1. Check Student Model: Qwen2.5-7B-Instruct
if [ -d "${ML_DIR}/models/Qwen2.5-7B-Instruct" ] && [ -n "$(ls -A "${ML_DIR}/models/Qwen2.5-7B-Instruct" 2>/dev/null)" ]; then
echo " ✅ Student model ('Qwen2.5-7B-Instruct') found locally at ml/models/Qwen2.5-7B-Instruct. Skipping download."
elif [ -d "${ML_DIR}/models/Qwen3.5-9B" ] && [ -n "$(ls -A "${ML_DIR}/models/Qwen3.5-9B" 2>/dev/null)" ]; then
echo " ✅ Model found locally at ml/models/Qwen3.5-9B. Skipping download."
else
echo " 📥 Student model not found locally in ml/models/. Downloading unsloth/Qwen2.5-7B-Instruct via hf..."
if command -v hf &> /dev/null; then
hf download unsloth/Qwen2.5-7B-Instruct --local-dir "${ML_DIR}/models/Qwen2.5-7B-Instruct"
else
"${PYTHON_CMD}" -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='unsloth/Qwen2.5-7B-Instruct', local_dir='${ML_DIR}/models/Qwen2.5-7B-Instruct')"
fi
echo " ✅ Downloaded Qwen2.5-7B-Instruct successfully."
fi
# 2. Check Teacher Model: Qwen2-72B-Instruct-FP8
if [ -d "${ML_DIR}/models/Qwen2-72B-Instruct-FP8" ] && [ -n "$(ls -A "${ML_DIR}/models/Qwen2-72B-Instruct-FP8" 2>/dev/null)" ]; then
echo " ✅ Teacher model ('Qwen2-72B-Instruct-FP8') found locally at ml/models/Qwen2-72B-Instruct-FP8. Skipping download."
elif [ -d "${ML_DIR}/models/Qwen2.5-72B-Instruct-FP8" ] && [ -n "$(ls -A "${ML_DIR}/models/Qwen2.5-72B-Instruct-FP8" 2>/dev/null)" ]; then
echo " ✅ Teacher model found locally at ml/models/Qwen2.5-72B-Instruct-FP8. Skipping download."
else
echo " 📥 Teacher model not found locally in ml/models/. Downloading Qwen/Qwen2-72B-Instruct-FP8 via hf..."
if command -v hf &> /dev/null; then
hf download Qwen/Qwen2-72B-Instruct-FP8 --local-dir "${ML_DIR}/models/Qwen2-72B-Instruct-FP8"
else
"${PYTHON_CMD}" -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='Qwen/Qwen2-72B-Instruct-FP8', local_dir='${ML_DIR}/models/Qwen2-72B-Instruct-FP8')"
fi
echo " ✅ Downloaded teacher model successfully."
fi
# 3. Check RAG Models: BAAI/bge-small-en-v1.5 and BAAI/bge-reranker-v2-m3
if [ -d "${ML_DIR}/models/bge-small-en-v1.5" ] && [ -n "$(ls -A "${ML_DIR}/models/bge-small-en-v1.5" 2>/dev/null)" ]; then
echo " ✅ RAG Embedding model ('bge-small-en-v1.5') found locally in ml/models/. Skipping download."
else
echo " 📥 RAG Embedding model not found locally in ml/models/. Downloading BAAI/bge-small-en-v1.5..."
if command -v hf &> /dev/null; then
hf download BAAI/bge-small-en-v1.5 --local-dir "${ML_DIR}/models/bge-small-en-v1.5"
else
"${PYTHON_CMD}" -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='BAAI/bge-small-en-v1.5', local_dir='${ML_DIR}/models/bge-small-en-v1.5')"
fi
echo " ✅ Downloaded RAG embedding model successfully."
fi
if [ -d "${ML_DIR}/models/bge-reranker-v2-m3" ] && [ -n "$(ls -A "${ML_DIR}/models/bge-reranker-v2-m3" 2>/dev/null)" ]; then
echo " ✅ RAG Reranker model ('bge-reranker-v2-m3') found locally in ml/models/. Skipping download."
else
echo " 📥 RAG Reranker model not found locally in ml/models/. Downloading BAAI/bge-reranker-v2-m3..."
if command -v hf &> /dev/null; then
hf download BAAI/bge-reranker-v2-m3 --local-dir "${ML_DIR}/models/bge-reranker-v2-m3"
else
"${PYTHON_CMD}" -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='BAAI/bge-reranker-v2-m3', local_dir='${ML_DIR}/models/bge-reranker-v2-m3')"
fi
echo " ✅ Downloaded RAG reranker model successfully."
fi
# ════════════════════════════════════════════════════════════════════════════════
# [PHASE 0.7/4]: RAG DEPENDENCY RESOLUTION
# ════════════════════════════════════════════════════════════════════════════════
echo -e "\n▶️ [PHASE 0.7/4]: Verifying Data Forge & Vector DB Dependencies..."
"${PYTHON_CMD}" -m pip install -q chromadb langchain-text-splitters rank_bm25 sentence-transformers langchain
echo " ✅ Dependencies synchronized."
cd "${ML_DIR}/data-forge"
# ════════════════════════════════════════════════════════════════════════════════
# [PHASE 1/4]: SYNTHETIC DATA FORGE
# ════════════════════════════════════════════════════════════════════════════════
if [ "${SKIP_GAN}" = true ]; then
echo -e "\n⏭️ [PHASE 1/4]: Skipping Synthetic GAN Forge (--skip-gan enabled)."
else
echo -e "\n▶️ [PHASE 1/4(a)]: Building Vector Database (ChromaDB)..."
"${PYTHON_CMD}" -u build_vector_db.py
echo -e "\n▶️ [PHASE 1/4(b)]: Forging Synthetic Data (GAN Forge via vLLM)..."
"${PYTHON_CMD}" -u gan_forge.py
fi
# ════════════════════════════════════════════════════════════════════════════════
# [PHASE 2/4]: DETERMINISTIC RUST DECISION TREE
# ════════════════════════════════════════════════════════════════════════════════
echo -e "\n▶️ [PHASE 2/4]: Building Deterministic Rust Decision Tree..."
"${PYTHON_CMD}" -u build_dpdp_tree.py
# ════════════════════════════════════════════════════════════════════════════════
# [PHASE 3/4]: UNSLOTH DATASET ALIGNMENT & FORMATTING
# ════════════════════════════════════════════════════════════════════════════════
echo -e "\n▶️ [PHASE 3/4]: Aligning & Formatting Data Schemas for Unsloth..."
"${PYTHON_CMD}" -u prepare_unsloth_data.py
echo -e "\n════════════════════════════════════════════════════════════════════════════════"
echo "✅ STAGE 1 COMPLETE: Models verified and Unsloth SFT & DPO training datasets synthesized."
echo "════════════════════════════════════════════════════════════════════════════════"
echo "📁 Execution logs saved to: ${LOG_DIR}"
echo "👉 Next step: Run 'bash ml/scripts/02_train_models.sh' to fine-tune both DPDP SLMs on DGX Spark."
|