Upload folder using huggingface_hub

qwen3_1.7b_4type_lora_old_format_dataset_a_4096_balanced_higher_maxm.yaml

@@ -0,0 +1,56 @@
+base_model: /home/wongshingyin/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+datasets:
+- path: /home/wongshingyin/deepseek_math_sft/ft_using_Axolotl_1/sft_4type_merged_n3to9_remap_completion_no_wrong_maxlen4096_rebalanced_higher_maxm.jsonl
+  ds_type: json
+  type:
+    field_instruction: input
+    field_input: __unused
+    field_output: output
+    format: "{instruction}"
+    no_input_format: "{instruction}"
+output_dir: ./outputs/qwen3-1.7b-sft-4type-old-format-lora-dataset-a-4096-balanced-higher-maxm
+dataset_prepared_path: /home/wongshingyin/deepseek_math_sft/ft_using_Axolotl_1/qwen3_1.7b_4type_prepared_4096_old_format_lora_dataset_a_balanced_higher_maxm
+micro_batch_size: 4
+gradient_accumulation_steps: 8
+# 508803 rows / (2*8*4_gpus) = 3975 steps per epoch
+# 1.0 fraction (2 epochs) = 15885 steps
+# 0.6 fraction = 9534 steps
+num_epochs: 2
+learning_rate: 0.00001
+lr_scheduler: cosine
+warmup_steps: 200
+optimizer: adamw_bnb_8bit
+weight_decay: 0.01
+val_set_size: 500
+eval_steps: 0.1
+early_stopping_patience: 3
+load_best_model_at_end: true
+bf16: true
+sequence_len: 4096
+pad_to_sequence_len: true
+group_by_length: false
+max_packed_sequence_len: null
+packing: true
+flash_attention: true
+gradient_checkpointing: true
+heads_k_stride: 1
+seed: 42
+data_seed: 42
+adapter: lora
+lora_r: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target_linear: true
+lora_target_modules:
+- q_proj
+- k_proj
+- v_proj
+- o_proj
+- gate_proj
+- up_proj
+- down_proj
+save_steps: 0.1
+save_total_limit: 10
+logging_steps: 200

qwen3_1.7b_training_paths.txt

@@ -0,0 +1,10 @@
+# Required File Paths for Qwen3-1.7B Phase 7 Training
+
+# 1. Base Model ID/Path (Local HF Cache)
+BASE_MODEL: /home/wongshingyin/.cache/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e
+
+# 2. Axolotl YAML Configuration File
+YAML_CONFIG: /home/wongshingyin/deepseek_math_sft/ft_using_Axolotl_1/qwen3_1.7b_4type_lora_old_format_dataset_a_4096_balanced_higher_maxm.yaml
+
+# 3. Training Dataset (Dataset A - 4096 Balanced)
+TRAINING_DATASET: /home/wongshingyin/deepseek_math_sft/ft_using_Axolotl_1/sft_4type_merged_n3to9_remap_completion_no_wrong_maxlen4096_rebalanced_higher_maxm.jsonl

run_qwen3_1.7b_lora_dataset_a.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Phase 7: Train 1.7B LoRA on Dataset A (4096 Balanced) with Seed 42
+# Stops at the shared training-budget fraction (0.6) = 9534 steps
+
+set -euo pipefail
+
+REPO_ROOT="/home/wongshingyin/deepseek_math_sft"
+SCRIPT_DIR="${REPO_ROOT}/ft_using_Axolotl_1"
+SESSION_NAME="${SESSION_NAME:-qwen3-1.7b-lora-dataset-a}"
+CONDA_ENV="${CONDA_ENV:-deepseek-math-sft}"
+
+YAML_PATH="${SCRIPT_DIR}/qwen3_1.7b_4type_lora_old_format_dataset_a_4096_balanced_higher_maxm.yaml"
+
+launch_tmux_if_needed() {
+  if [[ -n "${TMUX:-}" || "${INSIDE_TMUX_LAUNCH:-0}" == "1" ]]; then
+    return
+  fi
+  if tmux has-session -t "${SESSION_NAME}" 2>/dev/null; then
+    echo "tmux session already exists: ${SESSION_NAME}" >&2
+    echo "attach with: tmux attach -t ${SESSION_NAME}" >&2
+    exit 1
+  fi
+  tmux new-session -d -s "${SESSION_NAME}" \
+    "cd '${REPO_ROOT}' && INSIDE_TMUX_LAUNCH=1 SESSION_NAME='${SESSION_NAME}' CONDA_ENV='${CONDA_ENV}' CUDA_VISIBLE_DEVICES='${CUDA_VISIBLE_DEVICES:-0,1,2,3}' bash '${SCRIPT_DIR}/run_qwen3_1.7b_lora_dataset_a.sh'"
+  echo "launched tmux session: ${SESSION_NAME}"
+  echo "attach with: tmux attach -t ${SESSION_NAME}"
+  exit 0
+}
+
+activate_conda() {
+  if ! command -v conda >/dev/null 2>&1; then
+    echo "conda command not found; continuing without explicit activation" >&2
+    return
+  fi
+  local conda_base
+  conda_base="$(conda info --base)"
+  source "${conda_base}/etc/profile.d/conda.sh"
+  conda activate "${CONDA_ENV}"
+}
+
+main() {
+  launch_tmux_if_needed
+  cd "${REPO_ROOT}"
+  activate_conda
+
+  export AXOLOTL_DO_NOT_TRACK=1
+  # Default to GPUs 0,1,2,3 for 1.7B training
+  export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
+
+  echo "[$(date '+%F %T')] starting train 1.7B LoRA (Dataset A): ${YAML_PATH}"
+  axolotl train "${YAML_PATH}"
+  echo "[$(date '+%F %T')] finished train 1.7B LoRA"
+}
+
+main "$@"