Upload folder using huggingface_hub

viscop_qwen2.5_7b_viscop-lora_egocentric-expert/adapter_config.json

@@ -1,6 +1,6 @@
 {
   "auto_mapping": null,
-  "base_model_name_or_path": "/work/dreilly1/videollama3-image_7b_local",
+  "base_model_name_or_path": "/home/dreilly1/Projects/VisCoP/base_vlm/videollama3-image_7b_local",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -13,14 +13,6 @@
   "peft_type": "LORA",
   "r": 64,
   "revision": null,
-  "target_modules": [
-    "k_proj",
-    "o_proj",
-    "gate_proj",
-    "up_proj",
-    "q_proj",
-    "v_proj",
-    "down_proj"
-  ],
+  "target_modules": ".*model\\.layers\\..*\\.(gate_proj|o_proj|q_proj|v_proj|up_proj|down_proj|k_proj)$",
   "task_type": "CAUSAL_LM"
 }

viscop_qwen2.5_7b_viscop-lora_egocentric-expert/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:344525498c4532729f1abef575830691bde3db0846f3b0c1fdfa5bee74986da6
-size 323097987
+oid sha256:54b0d449306b48fc0193eeb2dec4280c5d91eb1180bc00faaf1e134edd727cbe
+size 323097578

viscop_qwen2.5_7b_viscop-lora_egocentric-expert/config.json

@@ -1,6 +1,6 @@
 {
   "_attn_implementation_autoset": true,
-  "_name_or_path": "./base_vlm/videollama3-image_7b_local/",
+  "_name_or_path": "/home/dreilly1/Projects/VisCoP/base_vlm/videollama3-image_7b_local",
   "architectures": [
     "Videollama3Qwen2ForCausalLM"
   ],
@@ -17,12 +17,15 @@
   "image_size": -1,
   "image_token_index": 151665,
   "image_token_length": 1,
+  "include_visual_probes": true,
   "include_visual_tokens": true,
   "initializer_range": 0.02,
+  "interaction_module": "cross_attention",
+  "interaction_module_layers": null,
   "intermediate_size": 18944,
   "is_alignment": false,
   "llm_lr": 1e-05,
-  "max_frames": 40,
+  "max_frames": 180,
   "max_position_embeddings": 32768,
   "max_window_layers": 28,
   "mm_hidden_size": 1152,
@@ -35,6 +38,8 @@
   "num_attention_heads": 28,
   "num_hidden_layers": 28,
   "num_key_value_heads": 4,
+  "num_visual_probes": 16,
+  "probe_token_index": 151668,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 1000000.0,
@@ -45,7 +50,6 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.46.3",
   "use_cache": true,
-  "use_learnable_queries": true,
   "use_mm_proj": true,
   "use_sliding_window": false,
   "use_token_compression": false,
@@ -124,9 +128,5 @@
     "use_bfloat16": false
   },
   "vision_encoder_lr": null,
-  "vocab_size": 152064,
-  "include_visual_probes": true,
-  "interaction_module": "cross_attention",
-  "interaction_module_layers": null,
-  "num_visual_probes": 16
-}
+  "vocab_size": 152064
+}

viscop_qwen2.5_7b_viscop-lora_egocentric-expert/non_lora_trainables.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b695fe812e07081c5af2397195980a2c50dd394a2cb705ec3beb768e8e359d4
-size 354951121
+oid sha256:bf012cd5517edf0873ba621833c3bb639f70191e9f9d733ff69e0bcdd71628be
+size 354950700

viscop_qwen2.5_7b_viscop-lora_egocentric-expert/train_viscop.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Environment Variables
+ARG_WORLD_SIZE=${1:-1}
+ARG_NPROC_PER_NODE=${2:-8}
+
+if [[ -v MASTER_ADDR_PASSED ]]; then
+    ARG_MASTER_ADDR=$MASTER_ADDR_PASSED # passed via slurm submission script
+else
+    ARG_MASTER_ADDR=127.0.0.1 # for dev environments
+fi
+ARG_MASTER_PORT=12355
+# ARG_RANK=$SLURM_NODEID
+ARG_RANK=0
+
+# Multiple conditions
+if [ ! -n "$WORLD_SIZE" ] || [ ! -n "$NPROC_PER_NODE" ]; then
+    WORLD_SIZE=$ARG_WORLD_SIZE
+    NPROC_PER_NODE=$ARG_NPROC_PER_NODE
+fi
+
+if [ ! -n "$MASTER_ADDR" ] || [ ! -n "$MASTER_PORT" ] || [ ! -n "$RANK" ]; then
+    MASTER_ADDR=$ARG_MASTER_ADDR
+    MASTER_PORT=$ARG_MASTER_PORT
+    RANK=$ARG_RANK
+fi
+
+echo "MASTER_ADDR: $MASTER_ADDR. MASTER_PORT: $MASTER_PORT. RANK: $RANK"
+echo "WORLD_SIZE: $WORLD_SIZE"
+echo "NPROC_PER_NODE: $NPROC_PER_NODE"
+
+# Training Arguments
+GLOBAL_BATCH_SIZE=128 # aka effective batch size
+LOCAL_BATCH_SIZE=8 # batch size per GPU
+GRADIENT_ACCUMULATION_STEPS=$[$GLOBAL_BATCH_SIZE/($WORLD_SIZE*$NPROC_PER_NODE*$LOCAL_BATCH_SIZE)]
+echo $GRADIENT_ACCUMULATION_STEPS
+
+INIT_MODEL=/home/dreilly1/Projects/VisCoP/base_vlm/videollama3-image_7b_local # path to base VLM (for ViSCoP we use VideoLLaMA3 as the base VLM)
+
+NUM_DATA_WORKERS=8
+NUM_TRAIN_EPOCHS=3
+LORA_TRAINING=True
+
+# ViSCoP Arguments
+NUM_VISUAL_PROBES=16
+INTERACTION_MODULE_POS=all
+PASS_PROBES_TO_LLM=True
+PASS_VIS_FEATURES_TO_LLM=True
+
+# Logging Arguments
+export WANDB_PROJECT=sony26_mm_viscop
+REPORT_TO=wandb
+OUTP_DIR=work_dirs/egoexo
+RUN_NAME=viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer
+
+# Data Arguments
+# DATA_DIR=/home/dreilly1/Projects/paired_egoexo_dl/paired_videos_DATA/
+DATA_DIR=/data/dreilly1/EgoExo4D_symlink/
+TRAINING_JSON="/home/dreilly1/Projects/viscop_sony/training_jsons/train-instr_viscop_egoview_ALLDATA.json"
+
+# if [[ $TRAINING_JSON == *"egoview"* ]]; then
+#     MAX_FRAMES=40 # use 40 frames for training on ego
+# else
+#     MAX_FRAMES=180
+# fi
+MAX_FRAMES=180
+
+# Optional Arguments. Set TESTING to 1 to quickly test the training script without logging or data workers, useful for debugging
+TESTING=0
+if [ $TESTING -eq 1 ]; then
+    NUM_DATA_WORKERS=0
+    REPORT_TO=none
+    RUN_NAME=TESTING
+fi
+
+mkdir -p "${OUTP_DIR}/${RUN_NAME}/"
+cp "$0" "${OUTP_DIR}/${RUN_NAME}/"
+
+torchrun --nnodes $WORLD_SIZE \
+    --nproc_per_node $NPROC_PER_NODE \
+    --master_addr=$MASTER_ADDR \
+    --master_port=$MASTER_PORT \
+    --node_rank $RANK \
+    viscop/train_viscop.py \
+    --interaction_module_layers $INTERACTION_MODULE_POS \
+    --lora_enable $LORA_TRAINING \
+    --num_train_epochs $NUM_TRAIN_EPOCHS \
+    --deepspeed scripts/zero2.json \
+    --model_type viscop_qwen2 \
+    --model_path $INIT_MODEL \
+    --vision_encoder DAMO-NLP-SG/SigLIP-NaViT \
+    --mm_projector_type mlp2x_gelu \
+    --data_path $TRAINING_JSON \
+    --data_folder $DATA_DIR \
+    --image_merge_size 2 \
+    --video_merge_size 2 \
+    --fps 1 \
+    --max_frames $MAX_FRAMES \
+    --model_max_length 16384 \
+    --mm_max_length 10240 \
+    --bf16 True \
+    --tf32 True \
+    --fp16 False \
+    --output_dir ${OUTP_DIR}/${RUN_NAME} \
+    --per_device_train_batch_size $LOCAL_BATCH_SIZE \
+    --per_device_eval_batch_size 2 \
+    --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS \
+    --evaluation_strategy "no" \
+    --save_strategy "no" \
+    --save_steps 5000 \
+    --save_total_limit 1 \
+    --mm_projector_lr 1e-5 \
+    --llm_lr 1e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers $NUM_DATA_WORKERS \
+    --report_to $REPORT_TO \
+    --run_name $RUN_NAME \
+    --dataset_cache_dir /home/dreilly1/.cache/viscop_datasetcache \
+    --include_visual_tokens $PASS_VIS_FEATURES_TO_LLM \
+    --include_visual_probes $PASS_PROBES_TO_LLM \
+    --num_visual_probes $NUM_VISUAL_PROBES