Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py +750 -0
- wandb/run-20260319_223122-agxw5i56/files/diff.patch +267 -0
- wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +267 -0
- wandb/run-20260319_223122-agxw5i56/files/output.log +26 -0
- wandb/run-20260319_223122-agxw5i56/files/requirements.txt +156 -0
- wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json +92 -0
- wandb/run-20260319_223122-agxw5i56/logs/debug-core.log +8 -0
- wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log +18 -0
- wandb/run-20260319_223122-agxw5i56/logs/debug.log +20 -0
- wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py +757 -0
- wandb/run-20260322_031609-y30zeekk/files/diff.patch +270 -0
- wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +270 -0
- wandb/run-20260322_031609-y30zeekk/files/output.log +26 -0
- wandb/run-20260322_031609-y30zeekk/files/requirements.txt +156 -0
- wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json +92 -0
- wandb/run-20260322_031609-y30zeekk/logs/debug-core.log +7 -0
- wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log +18 -0
- wandb/run-20260322_031609-y30zeekk/logs/debug.log +20 -0
- wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb +0 -0
- wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py +806 -0
- wandb/run-20260323_031014-fmwqi7oc/files/config.yaml +279 -0
- wandb/run-20260323_031014-fmwqi7oc/files/diff.patch +270 -0
- wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +270 -0
- wandb/run-20260323_031014-fmwqi7oc/files/output.log +82 -0
- wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt +156 -0
- wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json +93 -0
- wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json +1 -0
- wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log +15 -0
- wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log +29 -0
- wandb/run-20260323_031014-fmwqi7oc/logs/debug.log +22 -0
- wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb +0 -0
- wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py +815 -0
- wandb/run-20260324_164425-eb44q2tb/files/config.yaml +279 -0
- wandb/run-20260324_164425-eb44q2tb/files/diff.patch +308 -0
- wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +308 -0
- wandb/run-20260324_164425-eb44q2tb/files/output.log +42 -0
- wandb/run-20260324_164425-eb44q2tb/files/requirements.txt +156 -0
- wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json +93 -0
- wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json +1 -0
- wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log +15 -0
- wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log +29 -0
- wandb/run-20260324_164425-eb44q2tb/logs/debug.log +22 -0
- wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py +825 -0
- wandb/run-20260325_141344-o1lsxhwu/files/config.yaml +281 -0
- wandb/run-20260325_141344-o1lsxhwu/files/diff.patch +308 -0
- wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +308 -0
- wandb/run-20260325_141344-o1lsxhwu/files/output.log +31 -0
- wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt +156 -0
- wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json +93 -0
- wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json +1 -0
wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py
ADDED
|
@@ -0,0 +1,750 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 4 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
| 5 |
+
from datetime import timedelta
|
| 6 |
+
import json
|
| 7 |
+
import logging
|
| 8 |
+
import math
|
| 9 |
+
import shutil
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Dict, List, Any
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from torch.optim import AdamW
|
| 18 |
+
from torch.utils.data import Dataset, DataLoader
|
| 19 |
+
|
| 20 |
+
from omegaconf import OmegaConf
|
| 21 |
+
import wandb
|
| 22 |
+
|
| 23 |
+
from accelerate import Accelerator, InitProcessGroupKwargs
|
| 24 |
+
from accelerate.logging import get_logger
|
| 25 |
+
from accelerate.utils import set_seed
|
| 26 |
+
|
| 27 |
+
from train.utils import get_config, flatten_omega_conf, AverageMeter
|
| 28 |
+
from models.lr_schedulers import get_scheduler
|
| 29 |
+
from models.logging import set_verbosity_info, set_verbosity_error
|
| 30 |
+
|
| 31 |
+
# LLaDA-V / LLaVA
|
| 32 |
+
from transformers.utils import logging as hf_logging
|
| 33 |
+
hf_logging.set_verbosity_error()
|
| 34 |
+
|
| 35 |
+
from llava.model.builder import load_pretrained_model
|
| 36 |
+
from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord
|
| 37 |
+
from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
|
| 38 |
+
from llava.conversation import conv_templates
|
| 39 |
+
# from accelerate import DistributedDataParallelKwargs #添加
|
| 40 |
+
|
| 41 |
+
logger = get_logger(__name__, log_level="INFO")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def collapse_k_unique(lst, k: int):
|
| 45 |
+
if k <= 0:
|
| 46 |
+
raise ValueError("k must be > 0")
|
| 47 |
+
uniq = sorted(set(lst))
|
| 48 |
+
mapping = {}
|
| 49 |
+
n = len(uniq)
|
| 50 |
+
for idx, val in enumerate(uniq):
|
| 51 |
+
group = idx // k
|
| 52 |
+
end_idx = min((group + 1) * k - 1, n - 1)
|
| 53 |
+
rep = uniq[end_idx]
|
| 54 |
+
mapping[val] = rep
|
| 55 |
+
return [mapping[x] for x in lst]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
|
| 59 |
+
import copy
|
| 60 |
+
conv = copy.deepcopy(conv_templates["llava_llada"])
|
| 61 |
+
conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
|
| 62 |
+
conv.append_message(conv.roles[1], response_text)
|
| 63 |
+
return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
|
| 67 |
+
conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
|
| 68 |
+
ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
|
| 69 |
+
return ids.squeeze(0) # (L,)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
|
| 73 |
+
"""
|
| 74 |
+
统一处理 Image 和 Video
|
| 75 |
+
返回 (media_tensor_list, sizes, video_stats)
|
| 76 |
+
"""
|
| 77 |
+
from PIL import Image
|
| 78 |
+
from types import SimpleNamespace
|
| 79 |
+
|
| 80 |
+
if isinstance(model_cfg, dict):
|
| 81 |
+
model_cfg = SimpleNamespace(**model_cfg)
|
| 82 |
+
|
| 83 |
+
if media_type == "image":
|
| 84 |
+
image = Image.open(abs_path).convert("RGB")
|
| 85 |
+
if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
|
| 86 |
+
setattr(model_cfg, "image_aspect_ratio", "pad")
|
| 87 |
+
if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
|
| 88 |
+
setattr(model_cfg, "mm_patch_merge_type", "flat")
|
| 89 |
+
|
| 90 |
+
image_tensor_list = process_images([image], image_processor, model_cfg)
|
| 91 |
+
image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
|
| 92 |
+
image_sizes = [image.size]
|
| 93 |
+
return image_tensor_list, image_sizes, None
|
| 94 |
+
|
| 95 |
+
elif media_type == "video":
|
| 96 |
+
if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
|
| 97 |
+
if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 0)
|
| 98 |
+
if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", False)
|
| 99 |
+
|
| 100 |
+
# 使用官方提供的 decord 函数处理视频
|
| 101 |
+
video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
|
| 102 |
+
|
| 103 |
+
# 处理成 tensor
|
| 104 |
+
image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
|
| 105 |
+
image_tensor = image_tensor.to(dtype=dtype, device=device)
|
| 106 |
+
|
| 107 |
+
# 提取 size
|
| 108 |
+
if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
|
| 109 |
+
v_size = (video_frames.shape[2], video_frames.shape[1])
|
| 110 |
+
else:
|
| 111 |
+
v_size = (0, 0)
|
| 112 |
+
|
| 113 |
+
return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
|
| 117 |
+
out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
|
| 118 |
+
out[-x.numel():] = x
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
|
| 123 |
+
if x.size(0) == L:
|
| 124 |
+
return x
|
| 125 |
+
out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
|
| 126 |
+
out[-x.size(0):] = x
|
| 127 |
+
return out
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
|
| 131 |
+
"""通用媒体路径解析"""
|
| 132 |
+
p = sample.get(f"{media_type}_abs_path")
|
| 133 |
+
if p is None:
|
| 134 |
+
p = sample.get(media_type) or sample.get(f"{media_type}_path")
|
| 135 |
+
if p is None:
|
| 136 |
+
return None
|
| 137 |
+
if not os.path.isabs(p) and dataset_root:
|
| 138 |
+
p = os.path.join(dataset_root, p)
|
| 139 |
+
try:
|
| 140 |
+
return os.path.abspath(p)
|
| 141 |
+
except Exception:
|
| 142 |
+
return p
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class SFTVariantDatasetLLADAV(Dataset):
|
| 146 |
+
|
| 147 |
+
def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int):
|
| 148 |
+
super().__init__()
|
| 149 |
+
self.samples = samples
|
| 150 |
+
self.method = method
|
| 151 |
+
self.lower = lower
|
| 152 |
+
self.upper = upper
|
| 153 |
+
self.block_size = block_size
|
| 154 |
+
self.mask_times = mask_times
|
| 155 |
+
self.max_gen_length = max_gen_length
|
| 156 |
+
|
| 157 |
+
self.variants: List[dict] = []
|
| 158 |
+
self._build_variants()
|
| 159 |
+
|
| 160 |
+
def _build_variants(self):
|
| 161 |
+
sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
|
| 162 |
+
question_prompt="\nOnly give the best option."
|
| 163 |
+
|
| 164 |
+
rng = np.random.default_rng(seed=42)
|
| 165 |
+
for sid, s in enumerate(self.samples):
|
| 166 |
+
if self.method == "semi-ar":
|
| 167 |
+
step_map = s.get("step_map", None)
|
| 168 |
+
if step_map is None:
|
| 169 |
+
step_map = list(range(self.max_gen_length))
|
| 170 |
+
order_list = collapse_k_unique(step_map, self.block_size)
|
| 171 |
+
uniq_steps = sorted(set(order_list))
|
| 172 |
+
for st in uniq_steps:
|
| 173 |
+
self.variants.append(dict(
|
| 174 |
+
sample_id=sid,
|
| 175 |
+
prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt,
|
| 176 |
+
response=s["response"],
|
| 177 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 178 |
+
image=s.get("image", None),
|
| 179 |
+
image_path=s.get("image_path", None),
|
| 180 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 181 |
+
video=s.get("video", None),
|
| 182 |
+
video_path=s.get("video_path", None),
|
| 183 |
+
var_type="semi_ar",
|
| 184 |
+
var_param=int(st),
|
| 185 |
+
step_map=order_list,
|
| 186 |
+
mask_seed=None,
|
| 187 |
+
))
|
| 188 |
+
elif self.method == "random_masking":
|
| 189 |
+
for _ in range(self.mask_times):
|
| 190 |
+
t = float(rng.uniform(self.lower, self.upper))
|
| 191 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 192 |
+
self.variants.append(dict(
|
| 193 |
+
sample_id=sid,
|
| 194 |
+
prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt,
|
| 195 |
+
response=s["response"],
|
| 196 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 197 |
+
image=s.get("image", None),
|
| 198 |
+
image_path=s.get("image_path", None),
|
| 199 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 200 |
+
video=s.get("video", None),
|
| 201 |
+
video_path=s.get("video_path", None),
|
| 202 |
+
var_type="random_masking",
|
| 203 |
+
var_param=t,
|
| 204 |
+
step_map=None,
|
| 205 |
+
mask_seed=mask_seed,
|
| 206 |
+
))
|
| 207 |
+
else:
|
| 208 |
+
raise ValueError(f"Unknown SFT method: {self.method}")
|
| 209 |
+
|
| 210 |
+
def __len__(self):
|
| 211 |
+
return len(self.variants)
|
| 212 |
+
|
| 213 |
+
def __getitem__(self, idx):
|
| 214 |
+
v = self.variants[idx]
|
| 215 |
+
return {
|
| 216 |
+
"idx": idx,
|
| 217 |
+
"sample_id": v["sample_id"],
|
| 218 |
+
"prompt": v["prompt"],
|
| 219 |
+
"response": v["response"],
|
| 220 |
+
"image_abs_path": v.get("image_abs_path", None),
|
| 221 |
+
"image": v.get("image", None),
|
| 222 |
+
"image_path": v.get("image_path", None),
|
| 223 |
+
"video_abs_path": v.get("video_abs_path", None),
|
| 224 |
+
"video": v.get("video", None),
|
| 225 |
+
"video_path": v.get("video_path", None),
|
| 226 |
+
"var_type": v["var_type"],
|
| 227 |
+
"var_param": v["var_param"],
|
| 228 |
+
"step_map": v.get("step_map", None),
|
| 229 |
+
"mask_seed": v.get("mask_seed", None),
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
|
| 234 |
+
out: Dict[str, List[Any]] = {}
|
| 235 |
+
keys = batch[0].keys()
|
| 236 |
+
for k in keys:
|
| 237 |
+
out[k] = [b[k] for b in batch]
|
| 238 |
+
return out
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
@torch.no_grad()
|
| 242 |
+
def prepare_batch_noisy_embeds(
|
| 243 |
+
accelerator,
|
| 244 |
+
model,
|
| 245 |
+
tokenizer,
|
| 246 |
+
image_processor,
|
| 247 |
+
batch: Dict[str, List[Any]],
|
| 248 |
+
mask_id: int,
|
| 249 |
+
dataset_media_root: str | None,
|
| 250 |
+
) -> tuple | None:
|
| 251 |
+
|
| 252 |
+
device = accelerator.device
|
| 253 |
+
bsz = len(batch["idx"])
|
| 254 |
+
|
| 255 |
+
group: Dict[int, List[int]] = {}
|
| 256 |
+
for i, sid in enumerate(batch["sample_id"]):
|
| 257 |
+
group.setdefault(int(sid), []).append(i)
|
| 258 |
+
|
| 259 |
+
sample_cache: Dict[int, Dict[str, Any]] = {}
|
| 260 |
+
for sid, pos_list in group.items():
|
| 261 |
+
first_pos = pos_list[0]
|
| 262 |
+
prompt_text = batch["prompt"][first_pos]
|
| 263 |
+
response_text = batch["response"][first_pos]
|
| 264 |
+
|
| 265 |
+
sample_stub = {
|
| 266 |
+
"image_abs_path": batch.get("image_abs_path", [None])[first_pos],
|
| 267 |
+
"image": batch.get("image", [None])[first_pos],
|
| 268 |
+
"image_path": batch.get("image_path", [None])[first_pos],
|
| 269 |
+
"video_abs_path": batch.get("video_abs_path", [None])[first_pos],
|
| 270 |
+
"video": batch.get("video", [None])[first_pos],
|
| 271 |
+
"video_path": batch.get("video_path", [None])[first_pos],
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
# 优先检测视频,其次检测图片
|
| 275 |
+
media_type = None
|
| 276 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
|
| 277 |
+
if p_abs is not None:
|
| 278 |
+
media_type = "video"
|
| 279 |
+
else:
|
| 280 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
|
| 281 |
+
if p_abs is not None:
|
| 282 |
+
media_type = "image"
|
| 283 |
+
|
| 284 |
+
img_list, image_sizes, modalities = None, None, []
|
| 285 |
+
if p_abs is not None:
|
| 286 |
+
|
| 287 |
+
img_list, image_sizes, video_stats = load_and_process_media(
|
| 288 |
+
p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
modalities = [media_type]
|
| 292 |
+
|
| 293 |
+
# 若包含视频且需附加时间指引,将参数注入 prompt
|
| 294 |
+
if media_type == "video" and getattr(model.config, "add_time_instruction", False):
|
| 295 |
+
video_time, frame_time, num_frames = video_stats
|
| 296 |
+
time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
|
| 297 |
+
# 剔除原有可能存在的图片 Token 以免重叠
|
| 298 |
+
clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
|
| 299 |
+
prompt_text = f"{time_instruction}\n{clean_prompt}"
|
| 300 |
+
|
| 301 |
+
# Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN)
|
| 302 |
+
input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
|
| 303 |
+
|
| 304 |
+
end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
|
| 305 |
+
seq_len = len(end_header_seq)
|
| 306 |
+
pos_eh = -1
|
| 307 |
+
if seq_len > 0 and input_ids_text.numel() >= seq_len:
|
| 308 |
+
ids_list = input_ids_text.tolist()
|
| 309 |
+
for i in range(len(ids_list) - seq_len, -1, -1):
|
| 310 |
+
if ids_list[i:i + seq_len] == end_header_seq:
|
| 311 |
+
pos_eh = i + seq_len - 1
|
| 312 |
+
break
|
| 313 |
+
assert pos_eh != -1, " <|end_header_id|> not found"
|
| 314 |
+
start_pos = pos_eh + 1
|
| 315 |
+
|
| 316 |
+
labels_text = input_ids_text.clone()
|
| 317 |
+
labels_text[:start_pos] = -100
|
| 318 |
+
|
| 319 |
+
(_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
|
| 320 |
+
input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
|
| 321 |
+
)
|
| 322 |
+
clean_embeds = clean_embeds.squeeze(0) # (L, D)
|
| 323 |
+
new_labels = new_labels.squeeze(0) # (L,)
|
| 324 |
+
|
| 325 |
+
eot_id = 126348
|
| 326 |
+
eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
|
| 327 |
+
if eot_indices.numel() > 0:
|
| 328 |
+
last_eot_pos = eot_indices[-1, 0]
|
| 329 |
+
new_labels[last_eot_pos:] = -100
|
| 330 |
+
asst_mask = (new_labels != -100)
|
| 331 |
+
asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,)
|
| 332 |
+
|
| 333 |
+
sample_cache[sid] = dict(
|
| 334 |
+
clean_embeds=clean_embeds,
|
| 335 |
+
new_labels=new_labels,
|
| 336 |
+
asst_idx=asst_idx,
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
lengths = []
|
| 340 |
+
for i in range(bsz):
|
| 341 |
+
sid = int(batch["sample_id"][i])
|
| 342 |
+
lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0]))
|
| 343 |
+
Lmax = max(lengths)
|
| 344 |
+
D = next(iter(sample_cache.values()))["clean_embeds"].shape[1]
|
| 345 |
+
|
| 346 |
+
mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
|
| 347 |
+
|
| 348 |
+
noisy_list = []
|
| 349 |
+
labels_list = []
|
| 350 |
+
p_mask_list = []
|
| 351 |
+
attn_list = []
|
| 352 |
+
|
| 353 |
+
for i in range(bsz):
|
| 354 |
+
sid = int(batch["sample_id"][i])
|
| 355 |
+
var_type = batch["var_type"][i]
|
| 356 |
+
var_param = batch["var_param"][i]
|
| 357 |
+
step_map = batch["step_map"][i]
|
| 358 |
+
|
| 359 |
+
clean_embeds = sample_cache[sid]["clean_embeds"].clone()
|
| 360 |
+
new_labels = sample_cache[sid]["new_labels"].clone()
|
| 361 |
+
asst_idx = sample_cache[sid]["asst_idx"]
|
| 362 |
+
|
| 363 |
+
L = clean_embeds.size(0)
|
| 364 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 365 |
+
|
| 366 |
+
if var_type == "semi_ar":
|
| 367 |
+
order = torch.as_tensor(step_map, device=device)
|
| 368 |
+
M = asst_idx.numel()
|
| 369 |
+
if order.numel() != M:
|
| 370 |
+
m = min(M, order.numel())
|
| 371 |
+
asst_idx = asst_idx[:m]
|
| 372 |
+
order = order[:m]
|
| 373 |
+
order_full = torch.full((L,), -1, dtype=torch.long, device=device)
|
| 374 |
+
order_full[asst_idx] = order
|
| 375 |
+
step_val = int(var_param)
|
| 376 |
+
|
| 377 |
+
p = torch.empty(L, device=device).uniform_(0.0, 1.0)
|
| 378 |
+
lower = 0.0 + 1e-6
|
| 379 |
+
upper = 1.0 - 1e-6
|
| 380 |
+
thresh = (upper - lower) * torch.rand(1, device=device) + lower
|
| 381 |
+
block_mask = (order_full == step_val) & (p < thresh)
|
| 382 |
+
|
| 383 |
+
mask_pos = (order_full > step_val) | block_mask
|
| 384 |
+
noisy_embeds = clean_embeds.clone()
|
| 385 |
+
noisy_embeds[mask_pos] = mask_embed
|
| 386 |
+
|
| 387 |
+
pmask = block_mask
|
| 388 |
+
|
| 389 |
+
elif var_type == "random_masking":
|
| 390 |
+
M = asst_idx.numel()
|
| 391 |
+
if M == 0:
|
| 392 |
+
noisy_embeds = clean_embeds
|
| 393 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 394 |
+
else:
|
| 395 |
+
t = float(var_param)
|
| 396 |
+
mask_seed = batch.get("mask_seed", [None] * bsz)[i]
|
| 397 |
+
if mask_seed is None:
|
| 398 |
+
mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
|
| 399 |
+
gen = torch.Generator(device="cpu")
|
| 400 |
+
gen.manual_seed(int(mask_seed))
|
| 401 |
+
rnd_cpu = torch.rand(M, generator=gen) < t
|
| 402 |
+
rnd = rnd_cpu.to(device=device)
|
| 403 |
+
|
| 404 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 405 |
+
pmask[asst_idx] = rnd
|
| 406 |
+
|
| 407 |
+
noisy_embeds = clean_embeds.clone()
|
| 408 |
+
noisy_embeds[pmask] = mask_embed
|
| 409 |
+
|
| 410 |
+
else:
|
| 411 |
+
continue
|
| 412 |
+
|
| 413 |
+
if not pmask.any():
|
| 414 |
+
continue
|
| 415 |
+
# if not pmask.any():
|
| 416 |
+
# # print('*****no token masked*****')
|
| 417 |
+
# if asst_idx.numel() > 0:
|
| 418 |
+
# # 强制 Mask 最后一个 assistant token
|
| 419 |
+
# force_idx = asst_idx[-1]
|
| 420 |
+
# pmask[force_idx] = True
|
| 421 |
+
# noisy_embeds[force_idx] = mask_embed
|
| 422 |
+
# else:
|
| 423 |
+
# print("*********bad data*********")
|
| 424 |
+
# print(batch["sample_id"][i])
|
| 425 |
+
# # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过
|
| 426 |
+
# continue
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
|
| 430 |
+
labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
|
| 431 |
+
p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
|
| 432 |
+
attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
|
| 433 |
+
attn[-L:] = True
|
| 434 |
+
attn_list.append(attn)
|
| 435 |
+
|
| 436 |
+
if len(noisy_list) == 0:
|
| 437 |
+
return None
|
| 438 |
+
|
| 439 |
+
noisy_batch = torch.stack(noisy_list, dim=0)
|
| 440 |
+
labels_batch = torch.stack(labels_list, dim=0)
|
| 441 |
+
p_mask_batch = torch.stack(p_mask_list, dim=0)
|
| 442 |
+
attn_batch = torch.stack(attn_list, dim=0)
|
| 443 |
+
return noisy_batch, labels_batch, p_mask_batch, attn_batch
|
| 444 |
+
|
| 445 |
+
# ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ...
|
| 446 |
+
def save_checkpoint(model, tokenizer, config, accelerator, name):
|
| 447 |
+
output_dir = Path(config.experiment.project)
|
| 448 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 449 |
+
|
| 450 |
+
checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
|
| 451 |
+
if accelerator.is_main_process and checkpoints_total_limit is not None:
|
| 452 |
+
ckpts = sorted(
|
| 453 |
+
[d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
|
| 454 |
+
key=lambda p: int(p.name.split("-")[1]),
|
| 455 |
+
)
|
| 456 |
+
if len(ckpts) >= checkpoints_total_limit:
|
| 457 |
+
to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
|
| 458 |
+
logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
|
| 459 |
+
for p in to_remove:
|
| 460 |
+
shutil.rmtree(p, ignore_errors=True)
|
| 461 |
+
|
| 462 |
+
save_base = output_dir / "ckpt"
|
| 463 |
+
save_base.mkdir(exist_ok=True)
|
| 464 |
+
|
| 465 |
+
model_to_save = accelerator.unwrap_model(model)
|
| 466 |
+
state_dict = accelerator.get_state_dict(model)
|
| 467 |
+
|
| 468 |
+
if accelerator.is_main_process:
|
| 469 |
+
model_to_save.save_pretrained(
|
| 470 |
+
save_base / name,
|
| 471 |
+
save_function=accelerator.save,
|
| 472 |
+
state_dict=state_dict,
|
| 473 |
+
safe_serialization=True,
|
| 474 |
+
)
|
| 475 |
+
tokenizer.save_pretrained(str(save_base / name))
|
| 476 |
+
|
| 477 |
+
metadata = {
|
| 478 |
+
"save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 479 |
+
}
|
| 480 |
+
with (save_base / "metadata.json").open("w") as f:
|
| 481 |
+
json.dump(metadata, f, indent=2)
|
| 482 |
+
|
| 483 |
+
logger.info(f"Saved model + tokenizer to {save_base / name}")
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
def main():
|
| 487 |
+
config = get_config()
|
| 488 |
+
|
| 489 |
+
# TF32
|
| 490 |
+
if config.training.enable_tf32:
|
| 491 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 492 |
+
torch.backends.cudnn.benchmark = True
|
| 493 |
+
torch.backends.cudnn.deterministic = False
|
| 494 |
+
|
| 495 |
+
config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
|
| 496 |
+
|
| 497 |
+
# 🌟 实例化 DDP 处理器,允许存在未使用的参数
|
| 498 |
+
# ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
|
| 499 |
+
# process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600))
|
| 500 |
+
|
| 501 |
+
accelerator = Accelerator(
|
| 502 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 503 |
+
mixed_precision=config.training.mixed_precision,
|
| 504 |
+
log_with="wandb",
|
| 505 |
+
project_dir=config.experiment.logging_dir,
|
| 506 |
+
split_batches=True,
|
| 507 |
+
# kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去
|
| 508 |
+
# kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去
|
| 509 |
+
)
|
| 510 |
+
|
| 511 |
+
logging.basicConfig(
|
| 512 |
+
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
| 513 |
+
datefmt="%m/%d/%Y %H:%M:%S",
|
| 514 |
+
level=logging.INFO,
|
| 515 |
+
)
|
| 516 |
+
logger.info(accelerator.state, main_process_only=False)
|
| 517 |
+
if accelerator.is_local_main_process:
|
| 518 |
+
set_verbosity_info()
|
| 519 |
+
else:
|
| 520 |
+
set_verbosity_error()
|
| 521 |
+
|
| 522 |
+
if accelerator.is_main_process:
|
| 523 |
+
resume_wandb_run = config.wandb.resume
|
| 524 |
+
run_id = config.wandb.get("run_id", None)
|
| 525 |
+
if run_id is None:
|
| 526 |
+
resume_wandb_run = False
|
| 527 |
+
run_id = wandb.util.generate_id()
|
| 528 |
+
config.wandb.run_id = run_id
|
| 529 |
+
wandb_init_kwargs = dict(
|
| 530 |
+
name=config.experiment.project,
|
| 531 |
+
id=run_id,
|
| 532 |
+
resume=resume_wandb_run,
|
| 533 |
+
entity=config.wandb.get("entity", None),
|
| 534 |
+
config_exclude_keys=[],
|
| 535 |
+
)
|
| 536 |
+
wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
|
| 537 |
+
wandb_config.pop("experiment.resume_from_checkpoint", None)
|
| 538 |
+
|
| 539 |
+
accelerator.init_trackers(
|
| 540 |
+
config.experiment.project,
|
| 541 |
+
config=wandb_config,
|
| 542 |
+
init_kwargs={"wandb": wandb_init_kwargs},
|
| 543 |
+
)
|
| 544 |
+
|
| 545 |
+
if accelerator.is_main_process:
|
| 546 |
+
os.makedirs(config.experiment.project, exist_ok=True)
|
| 547 |
+
config_path = Path(config.experiment.project) / "config.yaml"
|
| 548 |
+
logging.info(f"Saving config to {config_path}")
|
| 549 |
+
OmegaConf.save(config, config_path)
|
| 550 |
+
|
| 551 |
+
if config.training.seed is not None:
|
| 552 |
+
set_seed(config.training.seed)
|
| 553 |
+
|
| 554 |
+
pretrained_model = config.model.pretrained_model
|
| 555 |
+
tokenizer, model, image_processor, _ = load_pretrained_model(
|
| 556 |
+
pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
|
| 557 |
+
)
|
| 558 |
+
model = model.to(accelerator.device)
|
| 559 |
+
|
| 560 |
+
special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
|
| 561 |
+
num_new = tokenizer.add_special_tokens(special_tokens)
|
| 562 |
+
if num_new > 0:
|
| 563 |
+
model.resize_token_embeddings(len(tokenizer))
|
| 564 |
+
if tokenizer.pad_token is None:
|
| 565 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 566 |
+
|
| 567 |
+
if getattr(config.training, "gradient_checkpointing_enable", False):
|
| 568 |
+
try:
|
| 569 |
+
model.enable_input_require_grads()
|
| 570 |
+
except Exception:
|
| 571 |
+
pass
|
| 572 |
+
try:
|
| 573 |
+
model.gradient_checkpointing_enable()
|
| 574 |
+
except Exception:
|
| 575 |
+
pass
|
| 576 |
+
|
| 577 |
+
mask_id = 126336
|
| 578 |
+
|
| 579 |
+
with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f:
|
| 580 |
+
data_list = json.load(f)
|
| 581 |
+
|
| 582 |
+
ds = SFTVariantDatasetLLADAV(
|
| 583 |
+
samples=data_list,
|
| 584 |
+
method=config.training.method,
|
| 585 |
+
lower=config.training.lower_p,
|
| 586 |
+
upper=config.training.upper_p,
|
| 587 |
+
block_size=config.training.block_size,
|
| 588 |
+
mask_times=config.training.mask_times_per_sample,
|
| 589 |
+
max_gen_length=config.training.max_gen_length
|
| 590 |
+
)
|
| 591 |
+
|
| 592 |
+
# Scheduler & DataLoader
|
| 593 |
+
total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
|
| 594 |
+
num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm)
|
| 595 |
+
num_train_epochs = config.training.num_train_epochs
|
| 596 |
+
max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
|
| 597 |
+
|
| 598 |
+
optimizer_config = config.optimizer.params
|
| 599 |
+
no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
|
| 600 |
+
optimizer_grouped_parameters = [
|
| 601 |
+
{
|
| 602 |
+
"params": [p for n, p in model.named_parameters() if
|
| 603 |
+
p.requires_grad and not any(nd in n for nd in no_decay)],
|
| 604 |
+
"weight_decay": optimizer_config.weight_decay,
|
| 605 |
+
},
|
| 606 |
+
{
|
| 607 |
+
"params": [p for n, p in model.named_parameters() if
|
| 608 |
+
p.requires_grad and any(nd in n for nd in no_decay)],
|
| 609 |
+
"weight_decay": 0.0,
|
| 610 |
+
},
|
| 611 |
+
]
|
| 612 |
+
optimizer = AdamW(
|
| 613 |
+
optimizer_grouped_parameters,
|
| 614 |
+
lr=optimizer_config.learning_rate,
|
| 615 |
+
betas=(optimizer_config.beta1, optimizer_config.beta2),
|
| 616 |
+
weight_decay=optimizer_config.weight_decay,
|
| 617 |
+
eps=optimizer_config.epsilon,
|
| 618 |
+
)
|
| 619 |
+
|
| 620 |
+
lr_scheduler = get_scheduler(
|
| 621 |
+
config.lr_scheduler.scheduler,
|
| 622 |
+
optimizer=optimizer,
|
| 623 |
+
num_training_steps=max_train_steps,
|
| 624 |
+
num_warmup_steps=config.lr_scheduler.params.warmup_steps,
|
| 625 |
+
min_lr_scale=config.lr_scheduler.params.min_lr_scale
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
dl = DataLoader(
|
| 629 |
+
ds,
|
| 630 |
+
batch_size=config.training.batch_size_lm,
|
| 631 |
+
sampler=None,
|
| 632 |
+
collate_fn=collate_variants,
|
| 633 |
+
num_workers=0,
|
| 634 |
+
)
|
| 635 |
+
|
| 636 |
+
model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl)
|
| 637 |
+
|
| 638 |
+
logger.info("***** Running LLaDA-V SFT training *****")
|
| 639 |
+
logger.info(f" Num variants = {len(ds)}")
|
| 640 |
+
logger.info(f" Num training steps = {max_train_steps}")
|
| 641 |
+
logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
|
| 642 |
+
logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
|
| 643 |
+
logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
|
| 644 |
+
|
| 645 |
+
data_time_m = AverageMeter()
|
| 646 |
+
end = time.time()
|
| 647 |
+
|
| 648 |
+
from tqdm.auto import tqdm
|
| 649 |
+
|
| 650 |
+
def forward_process_batch(batch):
|
| 651 |
+
out = prepare_batch_noisy_embeds(
|
| 652 |
+
accelerator=accelerator,
|
| 653 |
+
model=model,
|
| 654 |
+
tokenizer=tokenizer,
|
| 655 |
+
image_processor=image_processor,
|
| 656 |
+
batch=batch,
|
| 657 |
+
mask_id=mask_id,
|
| 658 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 659 |
+
)
|
| 660 |
+
if out is None:
|
| 661 |
+
return None
|
| 662 |
+
noisy_embeds, labels, p_mask, attn = out
|
| 663 |
+
|
| 664 |
+
outputs = model.get_model()(
|
| 665 |
+
inputs_embeds=noisy_embeds,
|
| 666 |
+
attention_mask=attn,
|
| 667 |
+
use_cache=False,
|
| 668 |
+
return_dict=True,
|
| 669 |
+
)
|
| 670 |
+
logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V)
|
| 671 |
+
|
| 672 |
+
# logits = outputs.logits.float() # (B, L, V)
|
| 673 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 674 |
+
|
| 675 |
+
safe_labels = labels.clone()
|
| 676 |
+
safe_labels[labels == -100] = 0
|
| 677 |
+
logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L)
|
| 678 |
+
|
| 679 |
+
loss_tok = - (logp_tok * p_mask).sum(dim=1)
|
| 680 |
+
num_mask = torch.clamp(p_mask.sum(dim=1), min=1)
|
| 681 |
+
loss = (loss_tok / num_mask).mean()
|
| 682 |
+
return loss
|
| 683 |
+
|
| 684 |
+
first_epoch = 0
|
| 685 |
+
grad_accum = config.training.gradient_accumulation_steps
|
| 686 |
+
|
| 687 |
+
for epoch in range(first_epoch, num_train_epochs):
|
| 688 |
+
model.train()
|
| 689 |
+
progress_bar = tqdm(
|
| 690 |
+
dl,
|
| 691 |
+
desc=f"Epoch {epoch+1}/{num_train_epochs}",
|
| 692 |
+
disable=not accelerator.is_local_main_process,
|
| 693 |
+
dynamic_ncols=True,
|
| 694 |
+
leave=True
|
| 695 |
+
)
|
| 696 |
+
|
| 697 |
+
# for step, batch in enumerate(progress_bar, start=1):
|
| 698 |
+
# data_time_m.update(time.time() - end)
|
| 699 |
+
|
| 700 |
+
# # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器
|
| 701 |
+
# with accelerator.accumulate(model):
|
| 702 |
+
# # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理
|
| 703 |
+
# loss = forward_process_batch(batch)
|
| 704 |
+
|
| 705 |
+
# accelerator.backward(loss)
|
| 706 |
+
|
| 707 |
+
# # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪
|
| 708 |
+
# if accelerator.sync_gradients:
|
| 709 |
+
# if config.training.max_grad_norm is not None:
|
| 710 |
+
# accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 711 |
+
|
| 712 |
+
# optimizer.step()
|
| 713 |
+
# lr_scheduler.step()
|
| 714 |
+
# optimizer.zero_grad(set_to_none=True)
|
| 715 |
+
|
| 716 |
+
# # 警告:坚决删掉这里的 torch.cuda.empty_cache()
|
| 717 |
+
|
| 718 |
+
step = 0
|
| 719 |
+
for step, batch in enumerate(progress_bar, start=1):
|
| 720 |
+
data_time_m.update(time.time() - end)
|
| 721 |
+
|
| 722 |
+
loss = forward_process_batch(batch)
|
| 723 |
+
if loss is None:
|
| 724 |
+
continue
|
| 725 |
+
loss = loss / grad_accum
|
| 726 |
+
accelerator.backward(loss)
|
| 727 |
+
|
| 728 |
+
if step % grad_accum == 0:
|
| 729 |
+
if config.training.max_grad_norm is not None:
|
| 730 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 731 |
+
optimizer.step()
|
| 732 |
+
lr_scheduler.step()
|
| 733 |
+
optimizer.zero_grad(set_to_none=True)
|
| 734 |
+
torch.cuda.empty_cache()
|
| 735 |
+
|
| 736 |
+
if step % grad_accum != 0:
|
| 737 |
+
if config.training.max_grad_norm is not None:
|
| 738 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 739 |
+
optimizer.step()
|
| 740 |
+
lr_scheduler.step()
|
| 741 |
+
optimizer.zero_grad(set_to_none=True)
|
| 742 |
+
torch.cuda.empty_cache()
|
| 743 |
+
|
| 744 |
+
accelerator.wait_for_everyone()
|
| 745 |
+
save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
|
| 746 |
+
accelerator.end_training()
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
if __name__ == "__main__":
|
| 750 |
+
main()
|
wandb/run-20260319_223122-agxw5i56/files/diff.patch
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
|
| 2 |
+
index 0ca6138..4c35ee9 100644
|
| 3 |
+
--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
|
| 4 |
+
+++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
|
| 5 |
+
@@ -10,7 +10,7 @@ downcast_bf16: 'no'
|
| 6 |
+
main_training_function: main
|
| 7 |
+
mixed_precision: bf16
|
| 8 |
+
num_machines: 1
|
| 9 |
+
-num_processes: 8
|
| 10 |
+
+num_processes: 4
|
| 11 |
+
rdzv_backend: static
|
| 12 |
+
same_network: true
|
| 13 |
+
tpu_env: []
|
| 14 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 15 |
+
index 3e07fbd..337824b 100644
|
| 16 |
+
--- a/configs/lladav_eval.yaml
|
| 17 |
+
+++ b/configs/lladav_eval.yaml
|
| 18 |
+
@@ -7,17 +7,17 @@ system:
|
| 19 |
+
HTTP_PROXY: null
|
| 20 |
+
HF_HOME: null
|
| 21 |
+
env_name: ""
|
| 22 |
+
- envs_dir: "/abs/path/of/envs"
|
| 23 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 24 |
+
+ envs_dir: null
|
| 25 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 26 |
+
|
| 27 |
+
-model: "/abs/path/of/model"
|
| 28 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 29 |
+
model_base: "lladav"
|
| 30 |
+
answer_must_in_box: True
|
| 31 |
+
|
| 32 |
+
dataset:
|
| 33 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 34 |
+
data_type: "mmu"
|
| 35 |
+
- image_root: "/abs/path/of/data/images"
|
| 36 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 37 |
+
|
| 38 |
+
execute:
|
| 39 |
+
num_chunk: 128
|
| 40 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 41 |
+
further_horizon: 128
|
| 42 |
+
output_unmasking_history: True
|
| 43 |
+
use_cache: True
|
| 44 |
+
-
|
| 45 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 46 |
+
index 7e38050..a07f620 100644
|
| 47 |
+
--- a/configs/rl_lladav.yaml
|
| 48 |
+
+++ b/configs/rl_lladav.yaml
|
| 49 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 50 |
+
system:
|
| 51 |
+
HTTP_PROXY: null
|
| 52 |
+
HF_HOME: null
|
| 53 |
+
- env_name: "env_name"
|
| 54 |
+
- envs_dir: "/abs/path/of/envs"
|
| 55 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 56 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 57 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 58 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 59 |
+
|
| 60 |
+
experiment:
|
| 61 |
+
project: "rl_lladav"
|
| 62 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 63 |
+
node_index: 0
|
| 64 |
+
|
| 65 |
+
model:
|
| 66 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 67 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 68 |
+
optimized_name: "optimized"
|
| 69 |
+
model_base: "lladav"
|
| 70 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 71 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 72 |
+
image_resolution: 512
|
| 73 |
+
|
| 74 |
+
dataset:
|
| 75 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 76 |
+
optimization_data: "rl_data"
|
| 77 |
+
data_type: "mmu"
|
| 78 |
+
- image_root: "/abs/path/of/data/images"
|
| 79 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 80 |
+
|
| 81 |
+
rollout:
|
| 82 |
+
num_task_per_step: 128
|
| 83 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 84 |
+
index 6557210..001a6e2 100644
|
| 85 |
+
--- a/configs/sft_lladav.yaml
|
| 86 |
+
+++ b/configs/sft_lladav.yaml
|
| 87 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 88 |
+
num_node: 1
|
| 89 |
+
|
| 90 |
+
model:
|
| 91 |
+
- pretrained_model: "/abs/path/of/model"
|
| 92 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 93 |
+
optimized_name: "optimized"
|
| 94 |
+
|
| 95 |
+
dataset:
|
| 96 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 97 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 98 |
+
data_type: "mmu"
|
| 99 |
+
- image_root: "/base_dir/data/images"
|
| 100 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 101 |
+
|
| 102 |
+
training:
|
| 103 |
+
gradient_checkpointing_enable: False
|
| 104 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 105 |
+
index e0b183f..7a08a39 100644
|
| 106 |
+
--- a/data/preprocess_sft.ipynb
|
| 107 |
+
+++ b/data/preprocess_sft.ipynb
|
| 108 |
+
@@ -30,15 +30,15 @@
|
| 109 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 110 |
+
"\n",
|
| 111 |
+
"# llada and mmada:\n",
|
| 112 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 113 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 114 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 115 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 116 |
+
"\n",
|
| 117 |
+
"# trado and sdar:\n",
|
| 118 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 119 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 120 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 121 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 122 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 123 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 124 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 125 |
+
"\n",
|
| 126 |
+
"\n",
|
| 127 |
+
"def get_prompt(data_i):\n",
|
| 128 |
+
@@ -67,6 +67,46 @@
|
| 129 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 130 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 131 |
+
]
|
| 132 |
+
+ },
|
| 133 |
+
+ {
|
| 134 |
+
+ "cell_type": "code",
|
| 135 |
+
+ "execution_count": 2,
|
| 136 |
+
+ "id": "32bbd872",
|
| 137 |
+
+ "metadata": {},
|
| 138 |
+
+ "outputs": [
|
| 139 |
+
+ {
|
| 140 |
+
+ "name": "stdout",
|
| 141 |
+
+ "output_type": "stream",
|
| 142 |
+
+ "text": [
|
| 143 |
+
+ "100\n"
|
| 144 |
+
+ ]
|
| 145 |
+
+ }
|
| 146 |
+
+ ],
|
| 147 |
+
+ "source": [
|
| 148 |
+
+ "import json\n",
|
| 149 |
+
+ "\n",
|
| 150 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 151 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 152 |
+
+ "save_data_list = []\n",
|
| 153 |
+
+ "save_num = 0\n",
|
| 154 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 155 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 156 |
+
+ " raw_data = json.load(fin)\n",
|
| 157 |
+
+ " for data in raw_data:\n",
|
| 158 |
+
+ " if save_num < 100:\n",
|
| 159 |
+
+ " save_data_list.append(data)\n",
|
| 160 |
+
+ " save_num += 1\n",
|
| 161 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 162 |
+
+ " print(len(save_data_list))\n"
|
| 163 |
+
+ ]
|
| 164 |
+
+ },
|
| 165 |
+
+ {
|
| 166 |
+
+ "cell_type": "code",
|
| 167 |
+
+ "execution_count": null,
|
| 168 |
+
+ "id": "fdfde30f",
|
| 169 |
+
+ "metadata": {},
|
| 170 |
+
+ "outputs": [],
|
| 171 |
+
+ "source": []
|
| 172 |
+
}
|
| 173 |
+
],
|
| 174 |
+
"metadata": {
|
| 175 |
+
@@ -85,7 +125,7 @@
|
| 176 |
+
"name": "python",
|
| 177 |
+
"nbconvert_exporter": "python",
|
| 178 |
+
"pygments_lexer": "ipython3",
|
| 179 |
+
- "version": "3.10.18"
|
| 180 |
+
+ "version": "3.10.19"
|
| 181 |
+
}
|
| 182 |
+
},
|
| 183 |
+
"nbformat": 4,
|
| 184 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 185 |
+
index 8f2c0f1..79d2e5b 100644
|
| 186 |
+
--- a/rl_v.py
|
| 187 |
+
+++ b/rl_v.py
|
| 188 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 189 |
+
def env_prefix(env) -> str:
|
| 190 |
+
return (
|
| 191 |
+
"source ~/.bashrc && "
|
| 192 |
+
- f"source activate {env} && "
|
| 193 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 194 |
+
+ "conda init && "
|
| 195 |
+
+ f"conda activate {env} && "
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
config = get_config()
|
| 199 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 200 |
+
index 4068fc8..d284a23 100644
|
| 201 |
+
--- a/sample/llava/conversation.py
|
| 202 |
+
+++ b/sample/llava/conversation.py
|
| 203 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 204 |
+
if self.version == "llama_v3":
|
| 205 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 206 |
+
elif self.version == "llava_llada":
|
| 207 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 208 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 211 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 212 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 213 |
+
index 3db5302..96e33b5 100644
|
| 214 |
+
--- a/sample/llava/model/builder.py
|
| 215 |
+
+++ b/sample/llava/model/builder.py
|
| 216 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 217 |
+
elif "llada" in model_name.lower():
|
| 218 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 219 |
+
|
| 220 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 221 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 222 |
+
+
|
| 223 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 224 |
+
if customized_config is None:
|
| 225 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 226 |
+
else:
|
| 227 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 228 |
+
index 4068fc8..d284a23 100644
|
| 229 |
+
--- a/train/llava/conversation.py
|
| 230 |
+
+++ b/train/llava/conversation.py
|
| 231 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 232 |
+
if self.version == "llama_v3":
|
| 233 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 234 |
+
elif self.version == "llava_llada":
|
| 235 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 236 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 237 |
+
else:
|
| 238 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 239 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 240 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 241 |
+
index 3db5302..aafab95 100644
|
| 242 |
+
--- a/train/llava/model/builder.py
|
| 243 |
+
+++ b/train/llava/model/builder.py
|
| 244 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 245 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 246 |
+
elif "llada" in model_name.lower():
|
| 247 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 248 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 249 |
+
|
| 250 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 251 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 252 |
+
if customized_config is None:
|
| 253 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 254 |
+
else:
|
| 255 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 256 |
+
index 3da28dc..8b17da3 100644
|
| 257 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 258 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 259 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 260 |
+
dpo_forward: Optional[bool] = None,
|
| 261 |
+
cache_position=None,
|
| 262 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 263 |
+
-
|
| 264 |
+
+ conversation_ids = None
|
| 265 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 266 |
+
# donate multi-dialogue
|
| 267 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
|
| 2 |
+
index 0ca6138..4c35ee9 100644
|
| 3 |
+
--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
|
| 4 |
+
+++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
|
| 5 |
+
@@ -10,7 +10,7 @@ downcast_bf16: 'no'
|
| 6 |
+
main_training_function: main
|
| 7 |
+
mixed_precision: bf16
|
| 8 |
+
num_machines: 1
|
| 9 |
+
-num_processes: 8
|
| 10 |
+
+num_processes: 4
|
| 11 |
+
rdzv_backend: static
|
| 12 |
+
same_network: true
|
| 13 |
+
tpu_env: []
|
| 14 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 15 |
+
index 3e07fbd..337824b 100644
|
| 16 |
+
--- a/configs/lladav_eval.yaml
|
| 17 |
+
+++ b/configs/lladav_eval.yaml
|
| 18 |
+
@@ -7,17 +7,17 @@ system:
|
| 19 |
+
HTTP_PROXY: null
|
| 20 |
+
HF_HOME: null
|
| 21 |
+
env_name: ""
|
| 22 |
+
- envs_dir: "/abs/path/of/envs"
|
| 23 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 24 |
+
+ envs_dir: null
|
| 25 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 26 |
+
|
| 27 |
+
-model: "/abs/path/of/model"
|
| 28 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 29 |
+
model_base: "lladav"
|
| 30 |
+
answer_must_in_box: True
|
| 31 |
+
|
| 32 |
+
dataset:
|
| 33 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 34 |
+
data_type: "mmu"
|
| 35 |
+
- image_root: "/abs/path/of/data/images"
|
| 36 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 37 |
+
|
| 38 |
+
execute:
|
| 39 |
+
num_chunk: 128
|
| 40 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 41 |
+
further_horizon: 128
|
| 42 |
+
output_unmasking_history: True
|
| 43 |
+
use_cache: True
|
| 44 |
+
-
|
| 45 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 46 |
+
index 7e38050..a07f620 100644
|
| 47 |
+
--- a/configs/rl_lladav.yaml
|
| 48 |
+
+++ b/configs/rl_lladav.yaml
|
| 49 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 50 |
+
system:
|
| 51 |
+
HTTP_PROXY: null
|
| 52 |
+
HF_HOME: null
|
| 53 |
+
- env_name: "env_name"
|
| 54 |
+
- envs_dir: "/abs/path/of/envs"
|
| 55 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 56 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 57 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 58 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 59 |
+
|
| 60 |
+
experiment:
|
| 61 |
+
project: "rl_lladav"
|
| 62 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 63 |
+
node_index: 0
|
| 64 |
+
|
| 65 |
+
model:
|
| 66 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 67 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 68 |
+
optimized_name: "optimized"
|
| 69 |
+
model_base: "lladav"
|
| 70 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 71 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 72 |
+
image_resolution: 512
|
| 73 |
+
|
| 74 |
+
dataset:
|
| 75 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 76 |
+
optimization_data: "rl_data"
|
| 77 |
+
data_type: "mmu"
|
| 78 |
+
- image_root: "/abs/path/of/data/images"
|
| 79 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 80 |
+
|
| 81 |
+
rollout:
|
| 82 |
+
num_task_per_step: 128
|
| 83 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 84 |
+
index 6557210..001a6e2 100644
|
| 85 |
+
--- a/configs/sft_lladav.yaml
|
| 86 |
+
+++ b/configs/sft_lladav.yaml
|
| 87 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 88 |
+
num_node: 1
|
| 89 |
+
|
| 90 |
+
model:
|
| 91 |
+
- pretrained_model: "/abs/path/of/model"
|
| 92 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 93 |
+
optimized_name: "optimized"
|
| 94 |
+
|
| 95 |
+
dataset:
|
| 96 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 97 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 98 |
+
data_type: "mmu"
|
| 99 |
+
- image_root: "/base_dir/data/images"
|
| 100 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 101 |
+
|
| 102 |
+
training:
|
| 103 |
+
gradient_checkpointing_enable: False
|
| 104 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 105 |
+
index e0b183f..7a08a39 100644
|
| 106 |
+
--- a/data/preprocess_sft.ipynb
|
| 107 |
+
+++ b/data/preprocess_sft.ipynb
|
| 108 |
+
@@ -30,15 +30,15 @@
|
| 109 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 110 |
+
"\n",
|
| 111 |
+
"# llada and mmada:\n",
|
| 112 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 113 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 114 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 115 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 116 |
+
"\n",
|
| 117 |
+
"# trado and sdar:\n",
|
| 118 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 119 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 120 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 121 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 122 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 123 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 124 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 125 |
+
"\n",
|
| 126 |
+
"\n",
|
| 127 |
+
"def get_prompt(data_i):\n",
|
| 128 |
+
@@ -67,6 +67,46 @@
|
| 129 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 130 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 131 |
+
]
|
| 132 |
+
+ },
|
| 133 |
+
+ {
|
| 134 |
+
+ "cell_type": "code",
|
| 135 |
+
+ "execution_count": 2,
|
| 136 |
+
+ "id": "32bbd872",
|
| 137 |
+
+ "metadata": {},
|
| 138 |
+
+ "outputs": [
|
| 139 |
+
+ {
|
| 140 |
+
+ "name": "stdout",
|
| 141 |
+
+ "output_type": "stream",
|
| 142 |
+
+ "text": [
|
| 143 |
+
+ "100\n"
|
| 144 |
+
+ ]
|
| 145 |
+
+ }
|
| 146 |
+
+ ],
|
| 147 |
+
+ "source": [
|
| 148 |
+
+ "import json\n",
|
| 149 |
+
+ "\n",
|
| 150 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 151 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 152 |
+
+ "save_data_list = []\n",
|
| 153 |
+
+ "save_num = 0\n",
|
| 154 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 155 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 156 |
+
+ " raw_data = json.load(fin)\n",
|
| 157 |
+
+ " for data in raw_data:\n",
|
| 158 |
+
+ " if save_num < 100:\n",
|
| 159 |
+
+ " save_data_list.append(data)\n",
|
| 160 |
+
+ " save_num += 1\n",
|
| 161 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 162 |
+
+ " print(len(save_data_list))\n"
|
| 163 |
+
+ ]
|
| 164 |
+
+ },
|
| 165 |
+
+ {
|
| 166 |
+
+ "cell_type": "code",
|
| 167 |
+
+ "execution_count": null,
|
| 168 |
+
+ "id": "fdfde30f",
|
| 169 |
+
+ "metadata": {},
|
| 170 |
+
+ "outputs": [],
|
| 171 |
+
+ "source": []
|
| 172 |
+
}
|
| 173 |
+
],
|
| 174 |
+
"metadata": {
|
| 175 |
+
@@ -85,7 +125,7 @@
|
| 176 |
+
"name": "python",
|
| 177 |
+
"nbconvert_exporter": "python",
|
| 178 |
+
"pygments_lexer": "ipython3",
|
| 179 |
+
- "version": "3.10.18"
|
| 180 |
+
+ "version": "3.10.19"
|
| 181 |
+
}
|
| 182 |
+
},
|
| 183 |
+
"nbformat": 4,
|
| 184 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 185 |
+
index 8f2c0f1..79d2e5b 100644
|
| 186 |
+
--- a/rl_v.py
|
| 187 |
+
+++ b/rl_v.py
|
| 188 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 189 |
+
def env_prefix(env) -> str:
|
| 190 |
+
return (
|
| 191 |
+
"source ~/.bashrc && "
|
| 192 |
+
- f"source activate {env} && "
|
| 193 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 194 |
+
+ "conda init && "
|
| 195 |
+
+ f"conda activate {env} && "
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
config = get_config()
|
| 199 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 200 |
+
index 4068fc8..d284a23 100644
|
| 201 |
+
--- a/sample/llava/conversation.py
|
| 202 |
+
+++ b/sample/llava/conversation.py
|
| 203 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 204 |
+
if self.version == "llama_v3":
|
| 205 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 206 |
+
elif self.version == "llava_llada":
|
| 207 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 208 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 211 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 212 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 213 |
+
index 3db5302..96e33b5 100644
|
| 214 |
+
--- a/sample/llava/model/builder.py
|
| 215 |
+
+++ b/sample/llava/model/builder.py
|
| 216 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 217 |
+
elif "llada" in model_name.lower():
|
| 218 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 219 |
+
|
| 220 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 221 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 222 |
+
+
|
| 223 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 224 |
+
if customized_config is None:
|
| 225 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 226 |
+
else:
|
| 227 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 228 |
+
index 4068fc8..d284a23 100644
|
| 229 |
+
--- a/train/llava/conversation.py
|
| 230 |
+
+++ b/train/llava/conversation.py
|
| 231 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 232 |
+
if self.version == "llama_v3":
|
| 233 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 234 |
+
elif self.version == "llava_llada":
|
| 235 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 236 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 237 |
+
else:
|
| 238 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 239 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 240 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 241 |
+
index 3db5302..aafab95 100644
|
| 242 |
+
--- a/train/llava/model/builder.py
|
| 243 |
+
+++ b/train/llava/model/builder.py
|
| 244 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 245 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 246 |
+
elif "llada" in model_name.lower():
|
| 247 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 248 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 249 |
+
|
| 250 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 251 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 252 |
+
if customized_config is None:
|
| 253 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 254 |
+
else:
|
| 255 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 256 |
+
index 3da28dc..8b17da3 100644
|
| 257 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 258 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 259 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 260 |
+
dpo_forward: Optional[bool] = None,
|
| 261 |
+
cache_position=None,
|
| 262 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 263 |
+
-
|
| 264 |
+
+ conversation_ids = None
|
| 265 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 266 |
+
# donate multi-dialogue
|
| 267 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
wandb/run-20260319_223122-agxw5i56/files/output.log
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
03/19/2026 22:31:26 - INFO - root - Saving config to sft_lladav_video_reversal_3_19/config.yaml
|
| 2 |
+
Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/LLaDA-V
|
| 3 |
+
Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
|
| 4 |
+
Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:20<00:00, 3.37s/it]
|
| 5 |
+
Rank 0: Model Class: LlavaLLaDAModelLM
|
| 6 |
+
/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
|
| 7 |
+
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
|
| 8 |
+
warnings.warn(
|
| 9 |
+
Before initializing optimizer states
|
| 10 |
+
MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB
|
| 11 |
+
CPU Virtual Memory: used = 208.51 GB, percent = 10.3%
|
| 12 |
+
After initializing optimizer states
|
| 13 |
+
MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB
|
| 14 |
+
CPU Virtual Memory: used = 228.52 GB, percent = 11.3%
|
| 15 |
+
After initializing ZeRO optimizer
|
| 16 |
+
MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB
|
| 17 |
+
CPU Virtual Memory: used = 229.31 GB, percent = 11.4%
|
| 18 |
+
03/19/2026 22:33:07 - INFO - __main__ - ***** Running LLaDA-V SFT training *****
|
| 19 |
+
03/19/2026 22:33:07 - INFO - __main__ - Num variants = 95996
|
| 20 |
+
03/19/2026 22:33:07 - INFO - __main__ - Num training steps = 3001
|
| 21 |
+
03/19/2026 22:33:07 - INFO - __main__ - Instantaneous batch size per device = 2
|
| 22 |
+
03/19/2026 22:33:07 - INFO - __main__ - Total train batch size (parallel * accum) = 32
|
| 23 |
+
03/19/2026 22:33:07 - INFO - __main__ - Gradient Accumulation steps = 4
|
| 24 |
+
Epoch 1/1: 0%| | 0/12000 [00:00<?, ?it/s]/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/checkpoint.py:87: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
|
| 25 |
+
warnings.warn(
|
| 26 |
+
Epoch 1/1: 44%|████████████████████████████████████████████████████████▋ | 5319/12000 [20:01:24<32:33:17, 17.54s/it]
|
wandb/run-20260319_223122-agxw5i56/files/requirements.txt
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==4.52.4
|
| 2 |
+
nvidia-nvtx-cu12==12.4.127
|
| 3 |
+
deepspeed==0.18.6
|
| 4 |
+
typing-inspection==0.4.2
|
| 5 |
+
scipy==1.15.3
|
| 6 |
+
MarkupSafe==3.0.3
|
| 7 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 8 |
+
yarl==1.22.0
|
| 9 |
+
tornado==6.5.4
|
| 10 |
+
nvidia-curand-cu12==10.3.5.147
|
| 11 |
+
nest-asyncio==1.6.0
|
| 12 |
+
safetensors==0.7.0
|
| 13 |
+
async-timeout==5.0.1
|
| 14 |
+
typing_extensions==4.15.0
|
| 15 |
+
asttokens==3.0.1
|
| 16 |
+
cuda-python==13.1.1
|
| 17 |
+
executing==2.2.1
|
| 18 |
+
nvidia-cudnn-frontend==1.18.0
|
| 19 |
+
comm==0.2.3
|
| 20 |
+
tokenizers==0.21.4
|
| 21 |
+
six==1.17.0
|
| 22 |
+
Pygments==2.19.2
|
| 23 |
+
nvidia-cusparselt-cu12==0.6.2
|
| 24 |
+
apache-tvm-ffi==0.1.8.post2
|
| 25 |
+
flash_attn==2.7.4.post1
|
| 26 |
+
tqdm==4.67.3
|
| 27 |
+
tensorboard-data-server==0.7.2
|
| 28 |
+
annotated-types==0.7.0
|
| 29 |
+
math-verify==0.9.0
|
| 30 |
+
idna==3.11
|
| 31 |
+
regex==2026.1.15
|
| 32 |
+
nvidia-cutlass-dsl-libs-base==4.4.0
|
| 33 |
+
flashinfer-python==0.6.3
|
| 34 |
+
click==8.3.1
|
| 35 |
+
pytz==2025.2
|
| 36 |
+
wheel==0.46.3
|
| 37 |
+
ipykernel==7.2.0
|
| 38 |
+
stack-data==0.6.3
|
| 39 |
+
absl-py==2.4.0
|
| 40 |
+
wandb==0.25.0
|
| 41 |
+
pandas==2.3.3
|
| 42 |
+
termcolor==3.3.0
|
| 43 |
+
sympy==1.13.1
|
| 44 |
+
joblib==1.5.3
|
| 45 |
+
prompt_toolkit==3.0.52
|
| 46 |
+
attrs==25.4.0
|
| 47 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 48 |
+
xxhash==3.6.0
|
| 49 |
+
aiohappyeyeballs==2.6.1
|
| 50 |
+
python-dateutil==2.9.0.post0
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nvidia-ml-py==13.590.48
|
| 53 |
+
multiprocess==0.70.18
|
| 54 |
+
decorator==5.2.1
|
| 55 |
+
cuda-pathfinder==1.3.4
|
| 56 |
+
multidict==6.7.1
|
| 57 |
+
pure_eval==0.2.3
|
| 58 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 59 |
+
omegaconf==2.3.0
|
| 60 |
+
PyYAML==6.0.3
|
| 61 |
+
debugpy==1.8.20
|
| 62 |
+
psutil==7.2.2
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 65 |
+
triton==3.2.0
|
| 66 |
+
antlr4-python3-runtime==4.9.3
|
| 67 |
+
peft==0.18.1
|
| 68 |
+
aiosignal==1.4.0
|
| 69 |
+
accelerate==1.12.0
|
| 70 |
+
liger_kernel==0.7.0
|
| 71 |
+
anyio==4.12.1
|
| 72 |
+
decord==0.6.0
|
| 73 |
+
h11==0.16.0
|
| 74 |
+
urllib3==2.6.3
|
| 75 |
+
wcwidth==0.6.0
|
| 76 |
+
networkx==3.4.2
|
| 77 |
+
huggingface_hub==0.36.2
|
| 78 |
+
tzdata==2025.3
|
| 79 |
+
xformers==0.0.29.post3
|
| 80 |
+
tabulate==0.9.0
|
| 81 |
+
certifi==2026.1.4
|
| 82 |
+
tensorboard==2.20.0
|
| 83 |
+
matplotlib-inline==0.2.1
|
| 84 |
+
pip==26.0.1
|
| 85 |
+
smmap==5.0.2
|
| 86 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 87 |
+
Markdown==3.10.2
|
| 88 |
+
nvidia-nccl-cu12==2.21.5
|
| 89 |
+
aiohttp==3.13.3
|
| 90 |
+
scikit-learn==1.7.2
|
| 91 |
+
nvidia-cutlass-dsl==4.4.0
|
| 92 |
+
setuptools==82.0.0
|
| 93 |
+
jupyter_core==5.9.1
|
| 94 |
+
asyncio==4.0.0
|
| 95 |
+
einops==0.8.2
|
| 96 |
+
pydantic==2.12.5
|
| 97 |
+
av==16.1.0
|
| 98 |
+
protobuf==6.33.5
|
| 99 |
+
evaluate==0.4.6
|
| 100 |
+
Jinja2==3.1.6
|
| 101 |
+
exceptiongroup==1.3.1
|
| 102 |
+
ipython==8.38.0
|
| 103 |
+
sentry-sdk==2.53.0
|
| 104 |
+
httpx==0.28.1
|
| 105 |
+
msgpack==1.1.2
|
| 106 |
+
platformdirs==4.9.2
|
| 107 |
+
hjson==3.1.0
|
| 108 |
+
torchvision==0.21.0
|
| 109 |
+
parso==0.8.6
|
| 110 |
+
cuda-bindings==13.1.1
|
| 111 |
+
pexpect==4.9.0
|
| 112 |
+
threadpoolctl==3.6.0
|
| 113 |
+
pyarrow==23.0.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 116 |
+
Werkzeug==3.1.5
|
| 117 |
+
fsspec==2025.10.0
|
| 118 |
+
pydantic_core==2.41.5
|
| 119 |
+
GitPython==3.1.46
|
| 120 |
+
propcache==0.4.1
|
| 121 |
+
mpmath==1.3.0
|
| 122 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 123 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 124 |
+
hf-xet==1.2.0
|
| 125 |
+
charset-normalizer==3.4.4
|
| 126 |
+
dill==0.4.0
|
| 127 |
+
datasets==4.5.0
|
| 128 |
+
py-cpuinfo==9.0.0
|
| 129 |
+
sentencepiece==0.2.1
|
| 130 |
+
latex2sympy2_extended==1.11.0
|
| 131 |
+
jupyter_client==8.8.0
|
| 132 |
+
pyzmq==27.1.0
|
| 133 |
+
numpy==2.2.6
|
| 134 |
+
gitdb==4.0.12
|
| 135 |
+
grpcio==1.78.0
|
| 136 |
+
packaging==26.0
|
| 137 |
+
requests==2.32.5
|
| 138 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 139 |
+
httpcore==1.0.9
|
| 140 |
+
traitlets==5.14.3
|
| 141 |
+
filelock==3.24.2
|
| 142 |
+
ptyprocess==0.7.0
|
| 143 |
+
pillow==12.1.1
|
| 144 |
+
torch==2.6.0
|
| 145 |
+
more-itertools==10.8.0
|
| 146 |
+
wheel==0.46.3
|
| 147 |
+
jaraco.text==4.0.0
|
| 148 |
+
zipp==3.23.0
|
| 149 |
+
autocommand==2.2.2
|
| 150 |
+
jaraco.context==6.1.0
|
| 151 |
+
jaraco.functools==4.4.0
|
| 152 |
+
tomli==2.4.0
|
| 153 |
+
backports.tarfile==1.2.0
|
| 154 |
+
importlib_metadata==8.7.1
|
| 155 |
+
platformdirs==4.4.0
|
| 156 |
+
packaging==26.0
|
wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-03-19T14:31:22.110949Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"config=configs/sft_lladav_video_reversal.yaml"
|
| 7 |
+
],
|
| 8 |
+
"program": "/data4/runqing/dLLM-RL/train/sft_lladav_video_reversal.py",
|
| 9 |
+
"codePath": "train/sft_lladav_video_reversal.py",
|
| 10 |
+
"codePathLocal": "train/sft_lladav_video_reversal.py",
|
| 11 |
+
"git": {
|
| 12 |
+
"remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
|
| 13 |
+
"commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
|
| 14 |
+
},
|
| 15 |
+
"email": "shixi_runqing@staff.weibo.com",
|
| 16 |
+
"root": "/data4/runqing/dLLM-RL",
|
| 17 |
+
"host": "b1301b38c75f",
|
| 18 |
+
"executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
|
| 19 |
+
"cpu_count": 96,
|
| 20 |
+
"cpu_count_logical": 192,
|
| 21 |
+
"gpu": "NVIDIA H800",
|
| 22 |
+
"gpu_count": 8,
|
| 23 |
+
"disk": {
|
| 24 |
+
"/": {
|
| 25 |
+
"total": "806578724864",
|
| 26 |
+
"used": "570999476224"
|
| 27 |
+
}
|
| 28 |
+
},
|
| 29 |
+
"memory": {
|
| 30 |
+
"total": "2164113813504"
|
| 31 |
+
},
|
| 32 |
+
"gpu_nvidia": [
|
| 33 |
+
{
|
| 34 |
+
"name": "NVIDIA H800",
|
| 35 |
+
"memoryTotal": "85520809984",
|
| 36 |
+
"cudaCores": 16896,
|
| 37 |
+
"architecture": "Hopper",
|
| 38 |
+
"uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"name": "NVIDIA H800",
|
| 42 |
+
"memoryTotal": "85520809984",
|
| 43 |
+
"cudaCores": 16896,
|
| 44 |
+
"architecture": "Hopper",
|
| 45 |
+
"uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"name": "NVIDIA H800",
|
| 49 |
+
"memoryTotal": "85520809984",
|
| 50 |
+
"cudaCores": 16896,
|
| 51 |
+
"architecture": "Hopper",
|
| 52 |
+
"uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"name": "NVIDIA H800",
|
| 56 |
+
"memoryTotal": "85520809984",
|
| 57 |
+
"cudaCores": 16896,
|
| 58 |
+
"architecture": "Hopper",
|
| 59 |
+
"uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "NVIDIA H800",
|
| 63 |
+
"memoryTotal": "85520809984",
|
| 64 |
+
"cudaCores": 16896,
|
| 65 |
+
"architecture": "Hopper",
|
| 66 |
+
"uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"name": "NVIDIA H800",
|
| 70 |
+
"memoryTotal": "85520809984",
|
| 71 |
+
"cudaCores": 16896,
|
| 72 |
+
"architecture": "Hopper",
|
| 73 |
+
"uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"name": "NVIDIA H800",
|
| 77 |
+
"memoryTotal": "85520809984",
|
| 78 |
+
"cudaCores": 16896,
|
| 79 |
+
"architecture": "Hopper",
|
| 80 |
+
"uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "NVIDIA H800",
|
| 84 |
+
"memoryTotal": "85520809984",
|
| 85 |
+
"cudaCores": 16896,
|
| 86 |
+
"architecture": "Hopper",
|
| 87 |
+
"uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
"cudaVersion": "12.4",
|
| 91 |
+
"writerId": "e2na0by3kxag1d04jgsxtaqowuonipoi"
|
| 92 |
+
}
|
wandb/run-20260319_223122-agxw5i56/logs/debug-core.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-19T22:31:22.229721003+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpec9x2qfv/port-3863399.txt","pid":3863399,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-03-19T22:31:22.232304613+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3863399}
|
| 3 |
+
{"time":"2026-03-19T22:31:22.2322714+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3863399-3864444-838468677/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-03-19T22:31:22.373663717+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-03-19T22:31:22.3828339+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"agxw5i56","id":"1(@)"}
|
| 6 |
+
{"time":"2026-03-19T22:31:26.60424022+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"agxw5i56","id":"1(@)"}
|
| 7 |
+
{"time":"2026-03-19T22:31:31.926890311+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6am2qm8cdzui"}
|
| 8 |
+
{"time":"2026-03-20T20:22:41.232415566+08:00","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-19T22:31:22.38308208+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-03-19T22:31:26.603517195+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
|
| 3 |
+
{"time":"2026-03-19T22:31:26.603666661+08:00","level":"INFO","msg":"stream: created new stream","id":"agxw5i56"}
|
| 4 |
+
{"time":"2026-03-19T22:31:26.604080689+08:00","level":"INFO","msg":"handler: started","stream_id":"agxw5i56"}
|
| 5 |
+
{"time":"2026-03-19T22:31:26.60422441+08:00","level":"INFO","msg":"stream: started","id":"agxw5i56"}
|
| 6 |
+
{"time":"2026-03-19T22:31:26.604248603+08:00","level":"INFO","msg":"writer: started","stream_id":"agxw5i56"}
|
| 7 |
+
{"time":"2026-03-19T22:31:26.604250045+08:00","level":"INFO","msg":"sender: started","stream_id":"agxw5i56"}
|
| 8 |
+
{"time":"2026-03-19T22:31:26.606722875+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
|
| 9 |
+
{"time":"2026-03-19T22:31:31.800015446+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a341b0ef13db33a9cfc8c700b1fd196501c0ae91a1f3805869f4c704a35346a3"}
|
| 10 |
+
{"time":"2026-03-19T22:31:31.800189317+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch</Resource><RequestId>189E4490C4AA826F</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a341b0ef13db33a9cfc8c700b1fd196501c0ae91a1f3805869f4c704a35346a3, Size: 11222}"}
|
| 11 |
+
{"time":"2026-03-19T22:31:31.801370376+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1bfe21a4d7c63fe36fba0cc6e7d8abc0349992391788b2eb567126807629b913"}
|
| 12 |
+
{"time":"2026-03-19T22:31:31.801479894+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt</Resource><RequestId>189E4490C4D2AC1E</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1bfe21a4d7c63fe36fba0cc6e7d8abc0349992391788b2eb567126807629b913, Size: 2966}"}
|
| 13 |
+
{"time":"2026-03-19T22:31:31.802152269+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=633c45789455deff40ac9ac290a8030f078715a583a3cc959d99dfeef370bdc2"}
|
| 14 |
+
{"time":"2026-03-19T22:31:31.802279633+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189E4490C44A5AF8</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=633c45789455deff40ac9ac290a8030f078715a583a3cc959d99dfeef370bdc2, Size: 11222}"}
|
| 15 |
+
{"time":"2026-03-19T22:31:31.803630426+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f087ddbd49f5feebf121bad34dc7ee132d5e81a79967564da3bbaee6ca6044cf"}
|
| 16 |
+
{"time":"2026-03-19T22:31:31.803693079+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json</Resource><RequestId>189E4490C4F3B717</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f087ddbd49f5feebf121bad34dc7ee132d5e81a79967564da3bbaee6ca6044cf, Size: 2650}"}
|
| 17 |
+
{"time":"2026-03-19T22:31:31.810924488+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e90cb55cb9fe21539c6e1e8d4e3349ca70bfaa57d266471b6485dd8c5fdb2bfe"}
|
| 18 |
+
{"time":"2026-03-19T22:31:31.810989568+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py</Resource><RequestId>189E4490C4D0F1F3</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py, Name: code/train/sft_lladav_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e90cb55cb9fe21539c6e1e8d4e3349ca70bfaa57d266471b6485dd8c5fdb2bfe, Size: 29481}"}
|
wandb/run-20260319_223122-agxw5i56/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
|
| 2 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Configure stats pid to 3863399
|
| 3 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/logs/debug.log
|
| 5 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log
|
| 6 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {'code_path': 'code/train/sft_lladav_video_reversal.py'}}
|
| 9 |
+
2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-19 22:31:22,373 INFO MainThread:3863399 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-19 22:31:22,377 INFO MainThread:3863399 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-19 22:31:22,378 INFO MainThread:3863399 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-19 22:31:22,383 INFO MainThread:3863399 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-19 22:31:26,722 INFO MainThread:3863399 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-19 22:31:26,920 INFO MainThread:3863399 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-19 22:31:26,920 INFO MainThread:3863399 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-19 22:31:26,921 INFO MainThread:3863399 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-19 22:31:26,921 INFO MainThread:3863399 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-19 22:31:26,928 INFO MainThread:3863399 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-19 22:31:26,929 INFO MainThread:3863399 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'agxw5i56', 'experiment.project': 'sft_lladav_video_reversal_3_19', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lladav_video_reversal_3_19/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 4, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.1, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lladav_video_reversal.yaml'}
|
wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 4 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
| 5 |
+
from datetime import timedelta
|
| 6 |
+
import json
|
| 7 |
+
import logging
|
| 8 |
+
import math
|
| 9 |
+
import shutil
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Dict, List, Any
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from torch.optim import AdamW
|
| 18 |
+
from torch.utils.data import Dataset, DataLoader
|
| 19 |
+
|
| 20 |
+
from omegaconf import OmegaConf
|
| 21 |
+
import wandb
|
| 22 |
+
|
| 23 |
+
from accelerate import Accelerator, InitProcessGroupKwargs
|
| 24 |
+
from accelerate.logging import get_logger
|
| 25 |
+
from accelerate.utils import set_seed
|
| 26 |
+
|
| 27 |
+
from train.utils import get_config, flatten_omega_conf, AverageMeter
|
| 28 |
+
from models.lr_schedulers import get_scheduler
|
| 29 |
+
from models.logging import set_verbosity_info, set_verbosity_error
|
| 30 |
+
|
| 31 |
+
# LLaDA-V / LLaVA
|
| 32 |
+
from transformers.utils import logging as hf_logging
|
| 33 |
+
hf_logging.set_verbosity_error()
|
| 34 |
+
|
| 35 |
+
from llava.model.builder import load_pretrained_model
|
| 36 |
+
from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord
|
| 37 |
+
from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
|
| 38 |
+
from llava.conversation import conv_templates
|
| 39 |
+
# from accelerate import DistributedDataParallelKwargs #添加
|
| 40 |
+
|
| 41 |
+
logger = get_logger(__name__, log_level="INFO")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def collapse_k_unique(lst, k: int):
|
| 45 |
+
if k <= 0:
|
| 46 |
+
raise ValueError("k must be > 0")
|
| 47 |
+
uniq = sorted(set(lst))
|
| 48 |
+
mapping = {}
|
| 49 |
+
n = len(uniq)
|
| 50 |
+
for idx, val in enumerate(uniq):
|
| 51 |
+
group = idx // k
|
| 52 |
+
end_idx = min((group + 1) * k - 1, n - 1)
|
| 53 |
+
rep = uniq[end_idx]
|
| 54 |
+
mapping[val] = rep
|
| 55 |
+
return [mapping[x] for x in lst]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
|
| 59 |
+
import copy
|
| 60 |
+
conv = copy.deepcopy(conv_templates["llava_llada"])
|
| 61 |
+
conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
|
| 62 |
+
conv.append_message(conv.roles[1], response_text)
|
| 63 |
+
return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
|
| 67 |
+
conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
|
| 68 |
+
ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
|
| 69 |
+
return ids.squeeze(0) # (L,)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
|
| 73 |
+
"""
|
| 74 |
+
统一处理 Image 和 Video
|
| 75 |
+
返回 (media_tensor_list, sizes, video_stats)
|
| 76 |
+
"""
|
| 77 |
+
from PIL import Image
|
| 78 |
+
from types import SimpleNamespace
|
| 79 |
+
|
| 80 |
+
if isinstance(model_cfg, dict):
|
| 81 |
+
model_cfg = SimpleNamespace(**model_cfg)
|
| 82 |
+
|
| 83 |
+
if media_type == "image":
|
| 84 |
+
image = Image.open(abs_path).convert("RGB")
|
| 85 |
+
if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
|
| 86 |
+
setattr(model_cfg, "image_aspect_ratio", "pad")
|
| 87 |
+
if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
|
| 88 |
+
setattr(model_cfg, "mm_patch_merge_type", "flat")
|
| 89 |
+
|
| 90 |
+
image_tensor_list = process_images([image], image_processor, model_cfg)
|
| 91 |
+
image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
|
| 92 |
+
image_sizes = [image.size]
|
| 93 |
+
return image_tensor_list, image_sizes, None
|
| 94 |
+
|
| 95 |
+
elif media_type == "video":
|
| 96 |
+
if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
|
| 97 |
+
if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16)
|
| 98 |
+
if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True)
|
| 99 |
+
|
| 100 |
+
# 使用官方提供的 decord 函数处理视频
|
| 101 |
+
video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
|
| 102 |
+
|
| 103 |
+
# 处理成 tensor
|
| 104 |
+
image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
|
| 105 |
+
image_tensor = image_tensor.to(dtype=dtype, device=device)
|
| 106 |
+
|
| 107 |
+
# 提取 size
|
| 108 |
+
if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
|
| 109 |
+
v_size = (video_frames.shape[2], video_frames.shape[1])
|
| 110 |
+
else:
|
| 111 |
+
v_size = (0, 0)
|
| 112 |
+
|
| 113 |
+
return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
|
| 117 |
+
out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
|
| 118 |
+
out[-x.numel():] = x
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
|
| 123 |
+
if x.size(0) == L:
|
| 124 |
+
return x
|
| 125 |
+
out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
|
| 126 |
+
out[-x.size(0):] = x
|
| 127 |
+
return out
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
|
| 131 |
+
"""通用媒体路径解析"""
|
| 132 |
+
p = sample.get(f"{media_type}_abs_path")
|
| 133 |
+
if p is None:
|
| 134 |
+
p = sample.get(media_type) or sample.get(f"{media_type}_path")
|
| 135 |
+
if p is None:
|
| 136 |
+
return None
|
| 137 |
+
if not os.path.isabs(p) and dataset_root:
|
| 138 |
+
p = os.path.join(dataset_root, p)
|
| 139 |
+
try:
|
| 140 |
+
return os.path.abspath(p)
|
| 141 |
+
except Exception:
|
| 142 |
+
return p
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class SFTVariantDatasetLLADAV(Dataset):
|
| 146 |
+
|
| 147 |
+
def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int):
|
| 148 |
+
super().__init__()
|
| 149 |
+
self.samples = samples
|
| 150 |
+
self.method = method
|
| 151 |
+
self.lower = lower
|
| 152 |
+
self.upper = upper
|
| 153 |
+
self.block_size = block_size
|
| 154 |
+
self.mask_times = mask_times
|
| 155 |
+
self.max_gen_length = max_gen_length
|
| 156 |
+
|
| 157 |
+
self.variants: List[dict] = []
|
| 158 |
+
self._build_variants()
|
| 159 |
+
|
| 160 |
+
def _build_variants(self):
|
| 161 |
+
sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
|
| 162 |
+
question_prompt="\nOnly give the best option."
|
| 163 |
+
multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
|
| 164 |
+
|
| 165 |
+
rng = np.random.default_rng(seed=42)
|
| 166 |
+
for sid, s in enumerate(self.samples):
|
| 167 |
+
if self.method == "semi-ar":
|
| 168 |
+
step_map = s.get("step_map", None)
|
| 169 |
+
if step_map is None:
|
| 170 |
+
step_map = list(range(self.max_gen_length))
|
| 171 |
+
order_list = collapse_k_unique(step_map, self.block_size)
|
| 172 |
+
uniq_steps = sorted(set(order_list))
|
| 173 |
+
for st in uniq_steps:
|
| 174 |
+
self.variants.append(dict(
|
| 175 |
+
sample_id=sid,
|
| 176 |
+
prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt,
|
| 177 |
+
response=s["response"],
|
| 178 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 179 |
+
image=s.get("image", None),
|
| 180 |
+
image_path=s.get("image_path", None),
|
| 181 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 182 |
+
video=s.get("video", None),
|
| 183 |
+
video_path=s.get("video_path", None),
|
| 184 |
+
var_type="semi_ar",
|
| 185 |
+
var_param=int(st),
|
| 186 |
+
step_map=order_list,
|
| 187 |
+
mask_seed=None,
|
| 188 |
+
))
|
| 189 |
+
elif self.method == "random_masking":
|
| 190 |
+
for _ in range(self.mask_times):
|
| 191 |
+
t = float(rng.uniform(self.lower, self.upper))
|
| 192 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 193 |
+
self.variants.append(dict(
|
| 194 |
+
sample_id=sid,
|
| 195 |
+
prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt,
|
| 196 |
+
response=s["response"],
|
| 197 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 198 |
+
image=s.get("image", None),
|
| 199 |
+
image_path=s.get("image_path", None),
|
| 200 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 201 |
+
video=s.get("video", None),
|
| 202 |
+
video_path=s.get("video_path", None),
|
| 203 |
+
var_type="random_masking",
|
| 204 |
+
var_param=t,
|
| 205 |
+
step_map=None,
|
| 206 |
+
mask_seed=mask_seed,
|
| 207 |
+
))
|
| 208 |
+
else:
|
| 209 |
+
raise ValueError(f"Unknown SFT method: {self.method}")
|
| 210 |
+
|
| 211 |
+
def __len__(self):
|
| 212 |
+
return len(self.variants)
|
| 213 |
+
|
| 214 |
+
def __getitem__(self, idx):
|
| 215 |
+
v = self.variants[idx]
|
| 216 |
+
return {
|
| 217 |
+
"idx": idx,
|
| 218 |
+
"sample_id": v["sample_id"],
|
| 219 |
+
"prompt": v["prompt"],
|
| 220 |
+
"response": v["response"],
|
| 221 |
+
"image_abs_path": v.get("image_abs_path", None),
|
| 222 |
+
"image": v.get("image", None),
|
| 223 |
+
"image_path": v.get("image_path", None),
|
| 224 |
+
"video_abs_path": v.get("video_abs_path", None),
|
| 225 |
+
"video": v.get("video", None),
|
| 226 |
+
"video_path": v.get("video_path", None),
|
| 227 |
+
"var_type": v["var_type"],
|
| 228 |
+
"var_param": v["var_param"],
|
| 229 |
+
"step_map": v.get("step_map", None),
|
| 230 |
+
"mask_seed": v.get("mask_seed", None),
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
|
| 235 |
+
out: Dict[str, List[Any]] = {}
|
| 236 |
+
keys = batch[0].keys()
|
| 237 |
+
for k in keys:
|
| 238 |
+
out[k] = [b[k] for b in batch]
|
| 239 |
+
return out
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
@torch.no_grad()
|
| 243 |
+
def prepare_batch_noisy_embeds(
|
| 244 |
+
accelerator,
|
| 245 |
+
model,
|
| 246 |
+
tokenizer,
|
| 247 |
+
image_processor,
|
| 248 |
+
batch: Dict[str, List[Any]],
|
| 249 |
+
mask_id: int,
|
| 250 |
+
dataset_media_root: str | None,
|
| 251 |
+
) -> tuple | None:
|
| 252 |
+
|
| 253 |
+
device = accelerator.device
|
| 254 |
+
bsz = len(batch["idx"])
|
| 255 |
+
|
| 256 |
+
group: Dict[int, List[int]] = {}
|
| 257 |
+
for i, sid in enumerate(batch["sample_id"]):
|
| 258 |
+
group.setdefault(int(sid), []).append(i)
|
| 259 |
+
|
| 260 |
+
sample_cache: Dict[int, Dict[str, Any]] = {}
|
| 261 |
+
for sid, pos_list in group.items():
|
| 262 |
+
first_pos = pos_list[0]
|
| 263 |
+
prompt_text = batch["prompt"][first_pos]
|
| 264 |
+
response_text = batch["response"][first_pos]
|
| 265 |
+
|
| 266 |
+
sample_stub = {
|
| 267 |
+
"image_abs_path": batch.get("image_abs_path", [None])[first_pos],
|
| 268 |
+
"image": batch.get("image", [None])[first_pos],
|
| 269 |
+
"image_path": batch.get("image_path", [None])[first_pos],
|
| 270 |
+
"video_abs_path": batch.get("video_abs_path", [None])[first_pos],
|
| 271 |
+
"video": batch.get("video", [None])[first_pos],
|
| 272 |
+
"video_path": batch.get("video_path", [None])[first_pos],
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
# 优先检测视频,其次检测图片
|
| 276 |
+
media_type = None
|
| 277 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
|
| 278 |
+
if p_abs is not None:
|
| 279 |
+
media_type = "video"
|
| 280 |
+
else:
|
| 281 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
|
| 282 |
+
if p_abs is not None:
|
| 283 |
+
media_type = "image"
|
| 284 |
+
|
| 285 |
+
img_list, image_sizes, modalities = None, None, []
|
| 286 |
+
if p_abs is not None:
|
| 287 |
+
|
| 288 |
+
img_list, image_sizes, video_stats = load_and_process_media(
|
| 289 |
+
p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
|
| 290 |
+
)
|
| 291 |
+
|
| 292 |
+
modalities = [media_type]
|
| 293 |
+
|
| 294 |
+
# 若包含视频且需附加时间指引,将参数注入 prompt
|
| 295 |
+
if media_type == "video" and getattr(model.config, "add_time_instruction", False):
|
| 296 |
+
video_time, frame_time, num_frames = video_stats
|
| 297 |
+
time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
|
| 298 |
+
# 剔除原有可能存在的图片 Token 以免重叠
|
| 299 |
+
clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
|
| 300 |
+
prompt_text = f"{time_instruction}\n{clean_prompt}"
|
| 301 |
+
|
| 302 |
+
# Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN)
|
| 303 |
+
input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
|
| 304 |
+
|
| 305 |
+
end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
|
| 306 |
+
seq_len = len(end_header_seq)
|
| 307 |
+
pos_eh = -1
|
| 308 |
+
if seq_len > 0 and input_ids_text.numel() >= seq_len:
|
| 309 |
+
ids_list = input_ids_text.tolist()
|
| 310 |
+
for i in range(len(ids_list) - seq_len, -1, -1):
|
| 311 |
+
if ids_list[i:i + seq_len] == end_header_seq:
|
| 312 |
+
pos_eh = i + seq_len - 1
|
| 313 |
+
break
|
| 314 |
+
assert pos_eh != -1, " <|end_header_id|> not found"
|
| 315 |
+
start_pos = pos_eh + 1
|
| 316 |
+
|
| 317 |
+
labels_text = input_ids_text.clone()
|
| 318 |
+
labels_text[:start_pos] = -100
|
| 319 |
+
|
| 320 |
+
(_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
|
| 321 |
+
input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
|
| 322 |
+
)
|
| 323 |
+
clean_embeds = clean_embeds.squeeze(0) # (L, D)
|
| 324 |
+
new_labels = new_labels.squeeze(0) # (L,)
|
| 325 |
+
|
| 326 |
+
eot_id = 126348
|
| 327 |
+
eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
|
| 328 |
+
if eot_indices.numel() > 0:
|
| 329 |
+
last_eot_pos = eot_indices[-1, 0]
|
| 330 |
+
new_labels[last_eot_pos:] = -100
|
| 331 |
+
asst_mask = (new_labels != -100)
|
| 332 |
+
asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,)
|
| 333 |
+
|
| 334 |
+
sample_cache[sid] = dict(
|
| 335 |
+
clean_embeds=clean_embeds,
|
| 336 |
+
new_labels=new_labels,
|
| 337 |
+
asst_idx=asst_idx,
|
| 338 |
+
)
|
| 339 |
+
|
| 340 |
+
lengths = []
|
| 341 |
+
for i in range(bsz):
|
| 342 |
+
sid = int(batch["sample_id"][i])
|
| 343 |
+
lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0]))
|
| 344 |
+
Lmax = max(lengths)
|
| 345 |
+
D = next(iter(sample_cache.values()))["clean_embeds"].shape[1]
|
| 346 |
+
|
| 347 |
+
mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
|
| 348 |
+
|
| 349 |
+
noisy_list = []
|
| 350 |
+
labels_list = []
|
| 351 |
+
p_mask_list = []
|
| 352 |
+
attn_list = []
|
| 353 |
+
|
| 354 |
+
for i in range(bsz):
|
| 355 |
+
sid = int(batch["sample_id"][i])
|
| 356 |
+
var_type = batch["var_type"][i]
|
| 357 |
+
var_param = batch["var_param"][i]
|
| 358 |
+
step_map = batch["step_map"][i]
|
| 359 |
+
|
| 360 |
+
clean_embeds = sample_cache[sid]["clean_embeds"].clone()
|
| 361 |
+
new_labels = sample_cache[sid]["new_labels"].clone()
|
| 362 |
+
asst_idx = sample_cache[sid]["asst_idx"]
|
| 363 |
+
|
| 364 |
+
L = clean_embeds.size(0)
|
| 365 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 366 |
+
|
| 367 |
+
if var_type == "semi_ar":
|
| 368 |
+
order = torch.as_tensor(step_map, device=device)
|
| 369 |
+
M = asst_idx.numel()
|
| 370 |
+
if order.numel() != M:
|
| 371 |
+
m = min(M, order.numel())
|
| 372 |
+
asst_idx = asst_idx[:m]
|
| 373 |
+
order = order[:m]
|
| 374 |
+
order_full = torch.full((L,), -1, dtype=torch.long, device=device)
|
| 375 |
+
order_full[asst_idx] = order
|
| 376 |
+
step_val = int(var_param)
|
| 377 |
+
|
| 378 |
+
p = torch.empty(L, device=device).uniform_(0.0, 1.0)
|
| 379 |
+
lower = 0.0 + 1e-6
|
| 380 |
+
upper = 1.0 - 1e-6
|
| 381 |
+
thresh = (upper - lower) * torch.rand(1, device=device) + lower
|
| 382 |
+
block_mask = (order_full == step_val) & (p < thresh)
|
| 383 |
+
|
| 384 |
+
mask_pos = (order_full > step_val) | block_mask
|
| 385 |
+
noisy_embeds = clean_embeds.clone()
|
| 386 |
+
noisy_embeds[mask_pos] = mask_embed
|
| 387 |
+
|
| 388 |
+
pmask = block_mask
|
| 389 |
+
|
| 390 |
+
elif var_type == "random_masking":
|
| 391 |
+
M = asst_idx.numel()
|
| 392 |
+
if M == 0:
|
| 393 |
+
noisy_embeds = clean_embeds
|
| 394 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 395 |
+
else:
|
| 396 |
+
t = float(var_param)
|
| 397 |
+
mask_seed = batch.get("mask_seed", [None] * bsz)[i]
|
| 398 |
+
if mask_seed is None:
|
| 399 |
+
mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
|
| 400 |
+
gen = torch.Generator(device="cpu")
|
| 401 |
+
gen.manual_seed(int(mask_seed))
|
| 402 |
+
rnd_cpu = torch.rand(M, generator=gen) < t
|
| 403 |
+
rnd = rnd_cpu.to(device=device)
|
| 404 |
+
|
| 405 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 406 |
+
pmask[asst_idx] = rnd
|
| 407 |
+
|
| 408 |
+
noisy_embeds = clean_embeds.clone()
|
| 409 |
+
noisy_embeds[pmask] = mask_embed
|
| 410 |
+
|
| 411 |
+
else:
|
| 412 |
+
continue
|
| 413 |
+
|
| 414 |
+
if not pmask.any():
|
| 415 |
+
if asst_idx.numel() > 0:
|
| 416 |
+
continue
|
| 417 |
+
else:
|
| 418 |
+
print("*********bad data*********")
|
| 419 |
+
print(batch["sample_id"][i])
|
| 420 |
+
continue
|
| 421 |
+
|
| 422 |
+
# if not pmask.any():
|
| 423 |
+
# print('*****no token masked*****')
|
| 424 |
+
# if asst_idx.numel() > 0:
|
| 425 |
+
# # 强制 Mask 最后一个 assistant token
|
| 426 |
+
# force_idx = asst_idx[-1]
|
| 427 |
+
# pmask[force_idx] = True
|
| 428 |
+
# noisy_embeds[force_idx] = mask_embed
|
| 429 |
+
# else:
|
| 430 |
+
# print("*********bad data*********")
|
| 431 |
+
# print(batch["sample_id"][i])
|
| 432 |
+
# # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过
|
| 433 |
+
# continue
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
|
| 437 |
+
labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
|
| 438 |
+
p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
|
| 439 |
+
attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
|
| 440 |
+
attn[-L:] = True
|
| 441 |
+
attn_list.append(attn)
|
| 442 |
+
|
| 443 |
+
if len(noisy_list) == 0:
|
| 444 |
+
return None
|
| 445 |
+
|
| 446 |
+
noisy_batch = torch.stack(noisy_list, dim=0)
|
| 447 |
+
labels_batch = torch.stack(labels_list, dim=0)
|
| 448 |
+
p_mask_batch = torch.stack(p_mask_list, dim=0)
|
| 449 |
+
attn_batch = torch.stack(attn_list, dim=0)
|
| 450 |
+
return noisy_batch, labels_batch, p_mask_batch, attn_batch
|
| 451 |
+
|
| 452 |
+
# ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ...
|
| 453 |
+
def save_checkpoint(model, tokenizer, config, accelerator, name):
|
| 454 |
+
output_dir = Path(config.experiment.project)
|
| 455 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 456 |
+
|
| 457 |
+
checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
|
| 458 |
+
if accelerator.is_main_process and checkpoints_total_limit is not None:
|
| 459 |
+
ckpts = sorted(
|
| 460 |
+
[d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
|
| 461 |
+
key=lambda p: int(p.name.split("-")[1]),
|
| 462 |
+
)
|
| 463 |
+
if len(ckpts) >= checkpoints_total_limit:
|
| 464 |
+
to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
|
| 465 |
+
logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
|
| 466 |
+
for p in to_remove:
|
| 467 |
+
shutil.rmtree(p, ignore_errors=True)
|
| 468 |
+
|
| 469 |
+
save_base = output_dir / "ckpt"
|
| 470 |
+
save_base.mkdir(exist_ok=True)
|
| 471 |
+
|
| 472 |
+
model_to_save = accelerator.unwrap_model(model)
|
| 473 |
+
state_dict = accelerator.get_state_dict(model)
|
| 474 |
+
|
| 475 |
+
if accelerator.is_main_process:
|
| 476 |
+
model_to_save.save_pretrained(
|
| 477 |
+
save_base / name,
|
| 478 |
+
save_function=accelerator.save,
|
| 479 |
+
state_dict=state_dict,
|
| 480 |
+
safe_serialization=True,
|
| 481 |
+
)
|
| 482 |
+
tokenizer.save_pretrained(str(save_base / name))
|
| 483 |
+
|
| 484 |
+
metadata = {
|
| 485 |
+
"save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 486 |
+
}
|
| 487 |
+
with (save_base / "metadata.json").open("w") as f:
|
| 488 |
+
json.dump(metadata, f, indent=2)
|
| 489 |
+
|
| 490 |
+
logger.info(f"Saved model + tokenizer to {save_base / name}")
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
def main():
|
| 494 |
+
config = get_config()
|
| 495 |
+
|
| 496 |
+
# TF32
|
| 497 |
+
if config.training.enable_tf32:
|
| 498 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 499 |
+
torch.backends.cudnn.benchmark = True
|
| 500 |
+
torch.backends.cudnn.deterministic = False
|
| 501 |
+
|
| 502 |
+
config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
|
| 503 |
+
|
| 504 |
+
# 🌟 实例化 DDP 处理器,允许存在未使用的参数
|
| 505 |
+
# ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
|
| 506 |
+
# process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600))
|
| 507 |
+
|
| 508 |
+
accelerator = Accelerator(
|
| 509 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 510 |
+
mixed_precision=config.training.mixed_precision,
|
| 511 |
+
log_with="wandb",
|
| 512 |
+
project_dir=config.experiment.logging_dir,
|
| 513 |
+
split_batches=True,
|
| 514 |
+
# kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去
|
| 515 |
+
# kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
logging.basicConfig(
|
| 519 |
+
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
| 520 |
+
datefmt="%m/%d/%Y %H:%M:%S",
|
| 521 |
+
level=logging.INFO,
|
| 522 |
+
)
|
| 523 |
+
logger.info(accelerator.state, main_process_only=False)
|
| 524 |
+
if accelerator.is_local_main_process:
|
| 525 |
+
set_verbosity_info()
|
| 526 |
+
else:
|
| 527 |
+
set_verbosity_error()
|
| 528 |
+
|
| 529 |
+
if accelerator.is_main_process:
|
| 530 |
+
resume_wandb_run = config.wandb.resume
|
| 531 |
+
run_id = config.wandb.get("run_id", None)
|
| 532 |
+
if run_id is None:
|
| 533 |
+
resume_wandb_run = False
|
| 534 |
+
run_id = wandb.util.generate_id()
|
| 535 |
+
config.wandb.run_id = run_id
|
| 536 |
+
wandb_init_kwargs = dict(
|
| 537 |
+
name=config.experiment.project,
|
| 538 |
+
id=run_id,
|
| 539 |
+
resume=resume_wandb_run,
|
| 540 |
+
entity=config.wandb.get("entity", None),
|
| 541 |
+
config_exclude_keys=[],
|
| 542 |
+
)
|
| 543 |
+
wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
|
| 544 |
+
wandb_config.pop("experiment.resume_from_checkpoint", None)
|
| 545 |
+
|
| 546 |
+
accelerator.init_trackers(
|
| 547 |
+
config.experiment.project,
|
| 548 |
+
config=wandb_config,
|
| 549 |
+
init_kwargs={"wandb": wandb_init_kwargs},
|
| 550 |
+
)
|
| 551 |
+
|
| 552 |
+
if accelerator.is_main_process:
|
| 553 |
+
os.makedirs(config.experiment.project, exist_ok=True)
|
| 554 |
+
config_path = Path(config.experiment.project) / "config.yaml"
|
| 555 |
+
logging.info(f"Saving config to {config_path}")
|
| 556 |
+
OmegaConf.save(config, config_path)
|
| 557 |
+
|
| 558 |
+
if config.training.seed is not None:
|
| 559 |
+
set_seed(config.training.seed)
|
| 560 |
+
|
| 561 |
+
pretrained_model = config.model.pretrained_model
|
| 562 |
+
tokenizer, model, image_processor, _ = load_pretrained_model(
|
| 563 |
+
pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
|
| 564 |
+
)
|
| 565 |
+
model = model.to(accelerator.device)
|
| 566 |
+
|
| 567 |
+
special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
|
| 568 |
+
num_new = tokenizer.add_special_tokens(special_tokens)
|
| 569 |
+
if num_new > 0:
|
| 570 |
+
model.resize_token_embeddings(len(tokenizer))
|
| 571 |
+
if tokenizer.pad_token is None:
|
| 572 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 573 |
+
|
| 574 |
+
if getattr(config.training, "gradient_checkpointing_enable", False):
|
| 575 |
+
try:
|
| 576 |
+
model.enable_input_require_grads()
|
| 577 |
+
except Exception:
|
| 578 |
+
pass
|
| 579 |
+
try:
|
| 580 |
+
model.gradient_checkpointing_enable()
|
| 581 |
+
except Exception:
|
| 582 |
+
pass
|
| 583 |
+
|
| 584 |
+
mask_id = 126336
|
| 585 |
+
|
| 586 |
+
with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f:
|
| 587 |
+
data_list = json.load(f)
|
| 588 |
+
|
| 589 |
+
ds = SFTVariantDatasetLLADAV(
|
| 590 |
+
samples=data_list,
|
| 591 |
+
method=config.training.method,
|
| 592 |
+
lower=config.training.lower_p,
|
| 593 |
+
upper=config.training.upper_p,
|
| 594 |
+
block_size=config.training.block_size,
|
| 595 |
+
mask_times=config.training.mask_times_per_sample,
|
| 596 |
+
max_gen_length=config.training.max_gen_length
|
| 597 |
+
)
|
| 598 |
+
|
| 599 |
+
# Scheduler & DataLoader
|
| 600 |
+
total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
|
| 601 |
+
num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm)
|
| 602 |
+
num_train_epochs = config.training.num_train_epochs
|
| 603 |
+
max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
|
| 604 |
+
|
| 605 |
+
optimizer_config = config.optimizer.params
|
| 606 |
+
no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
|
| 607 |
+
optimizer_grouped_parameters = [
|
| 608 |
+
{
|
| 609 |
+
"params": [p for n, p in model.named_parameters() if
|
| 610 |
+
p.requires_grad and not any(nd in n for nd in no_decay)],
|
| 611 |
+
"weight_decay": optimizer_config.weight_decay,
|
| 612 |
+
},
|
| 613 |
+
{
|
| 614 |
+
"params": [p for n, p in model.named_parameters() if
|
| 615 |
+
p.requires_grad and any(nd in n for nd in no_decay)],
|
| 616 |
+
"weight_decay": 0.0,
|
| 617 |
+
},
|
| 618 |
+
]
|
| 619 |
+
optimizer = AdamW(
|
| 620 |
+
optimizer_grouped_parameters,
|
| 621 |
+
lr=optimizer_config.learning_rate,
|
| 622 |
+
betas=(optimizer_config.beta1, optimizer_config.beta2),
|
| 623 |
+
weight_decay=optimizer_config.weight_decay,
|
| 624 |
+
eps=optimizer_config.epsilon,
|
| 625 |
+
)
|
| 626 |
+
|
| 627 |
+
lr_scheduler = get_scheduler(
|
| 628 |
+
config.lr_scheduler.scheduler,
|
| 629 |
+
optimizer=optimizer,
|
| 630 |
+
num_training_steps=max_train_steps,
|
| 631 |
+
num_warmup_steps=config.lr_scheduler.params.warmup_steps,
|
| 632 |
+
min_lr_scale=config.lr_scheduler.params.min_lr_scale
|
| 633 |
+
)
|
| 634 |
+
|
| 635 |
+
dl = DataLoader(
|
| 636 |
+
ds,
|
| 637 |
+
batch_size=config.training.batch_size_lm,
|
| 638 |
+
sampler=None,
|
| 639 |
+
collate_fn=collate_variants,
|
| 640 |
+
num_workers=0,
|
| 641 |
+
)
|
| 642 |
+
|
| 643 |
+
model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl)
|
| 644 |
+
|
| 645 |
+
logger.info("***** Running LLaDA-V SFT training *****")
|
| 646 |
+
logger.info(f" Num variants = {len(ds)}")
|
| 647 |
+
logger.info(f" Num training steps = {max_train_steps}")
|
| 648 |
+
logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
|
| 649 |
+
logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
|
| 650 |
+
logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
|
| 651 |
+
|
| 652 |
+
data_time_m = AverageMeter()
|
| 653 |
+
end = time.time()
|
| 654 |
+
|
| 655 |
+
from tqdm.auto import tqdm
|
| 656 |
+
|
| 657 |
+
def forward_process_batch(batch):
|
| 658 |
+
out = prepare_batch_noisy_embeds(
|
| 659 |
+
accelerator=accelerator,
|
| 660 |
+
model=model,
|
| 661 |
+
tokenizer=tokenizer,
|
| 662 |
+
image_processor=image_processor,
|
| 663 |
+
batch=batch,
|
| 664 |
+
mask_id=mask_id,
|
| 665 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 666 |
+
)
|
| 667 |
+
if out is None:
|
| 668 |
+
return None
|
| 669 |
+
noisy_embeds, labels, p_mask, attn = out
|
| 670 |
+
|
| 671 |
+
outputs = model.get_model()(
|
| 672 |
+
inputs_embeds=noisy_embeds,
|
| 673 |
+
attention_mask=attn,
|
| 674 |
+
use_cache=False,
|
| 675 |
+
return_dict=True,
|
| 676 |
+
)
|
| 677 |
+
logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V)
|
| 678 |
+
|
| 679 |
+
# logits = outputs.logits.float() # (B, L, V)
|
| 680 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 681 |
+
|
| 682 |
+
safe_labels = labels.clone()
|
| 683 |
+
safe_labels[labels == -100] = 0
|
| 684 |
+
logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L)
|
| 685 |
+
|
| 686 |
+
loss_tok = - (logp_tok * p_mask).sum(dim=1)
|
| 687 |
+
num_mask = torch.clamp(p_mask.sum(dim=1), min=1)
|
| 688 |
+
loss = (loss_tok / num_mask).mean()
|
| 689 |
+
return loss
|
| 690 |
+
|
| 691 |
+
first_epoch = 0
|
| 692 |
+
grad_accum = config.training.gradient_accumulation_steps
|
| 693 |
+
|
| 694 |
+
for epoch in range(first_epoch, num_train_epochs):
|
| 695 |
+
model.train()
|
| 696 |
+
progress_bar = tqdm(
|
| 697 |
+
dl,
|
| 698 |
+
desc=f"Epoch {epoch+1}/{num_train_epochs}",
|
| 699 |
+
disable=not accelerator.is_local_main_process,
|
| 700 |
+
dynamic_ncols=True,
|
| 701 |
+
leave=True
|
| 702 |
+
)
|
| 703 |
+
|
| 704 |
+
# for step, batch in enumerate(progress_bar, start=1):
|
| 705 |
+
# data_time_m.update(time.time() - end)
|
| 706 |
+
|
| 707 |
+
# # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器
|
| 708 |
+
# with accelerator.accumulate(model):
|
| 709 |
+
# # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理
|
| 710 |
+
# loss = forward_process_batch(batch)
|
| 711 |
+
|
| 712 |
+
# accelerator.backward(loss)
|
| 713 |
+
|
| 714 |
+
# # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪
|
| 715 |
+
# if accelerator.sync_gradients:
|
| 716 |
+
# if config.training.max_grad_norm is not None:
|
| 717 |
+
# accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 718 |
+
|
| 719 |
+
# optimizer.step()
|
| 720 |
+
# lr_scheduler.step()
|
| 721 |
+
# optimizer.zero_grad(set_to_none=True)
|
| 722 |
+
|
| 723 |
+
# # 警告:坚决删掉这里的 torch.cuda.empty_cache()
|
| 724 |
+
|
| 725 |
+
step = 0
|
| 726 |
+
for step, batch in enumerate(progress_bar, start=1):
|
| 727 |
+
data_time_m.update(time.time() - end)
|
| 728 |
+
|
| 729 |
+
loss = forward_process_batch(batch)
|
| 730 |
+
if loss is None:
|
| 731 |
+
continue
|
| 732 |
+
loss = loss / grad_accum
|
| 733 |
+
accelerator.backward(loss)
|
| 734 |
+
|
| 735 |
+
if step % grad_accum == 0:
|
| 736 |
+
if config.training.max_grad_norm is not None:
|
| 737 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 738 |
+
optimizer.step()
|
| 739 |
+
lr_scheduler.step()
|
| 740 |
+
optimizer.zero_grad(set_to_none=True)
|
| 741 |
+
torch.cuda.empty_cache()
|
| 742 |
+
|
| 743 |
+
if step % grad_accum != 0:
|
| 744 |
+
if config.training.max_grad_norm is not None:
|
| 745 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 746 |
+
optimizer.step()
|
| 747 |
+
lr_scheduler.step()
|
| 748 |
+
optimizer.zero_grad(set_to_none=True)
|
| 749 |
+
torch.cuda.empty_cache()
|
| 750 |
+
|
| 751 |
+
accelerator.wait_for_everyone()
|
| 752 |
+
save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
|
| 753 |
+
accelerator.end_training()
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
if __name__ == "__main__":
|
| 757 |
+
main()
|
wandb/run-20260322_031609-y30zeekk/files/diff.patch
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..5701c74 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,62 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [],
|
| 158 |
+
+ "source": [
|
| 159 |
+
+ "import json\n",
|
| 160 |
+
+ "\n",
|
| 161 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
|
| 162 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 163 |
+
+ "save_data_list = []\n",
|
| 164 |
+
+ "save_num = 0\n",
|
| 165 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 166 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 167 |
+
+ " raw_data = json.load(fin)\n",
|
| 168 |
+
+ " for data in raw_data:\n",
|
| 169 |
+
+ " if save_num < 100:\n",
|
| 170 |
+
+ " save_data_list.append(data)\n",
|
| 171 |
+
+ " save_num += 1\n",
|
| 172 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 173 |
+
+ " print(len(save_data_list))\n"
|
| 174 |
+
+ ]
|
| 175 |
+
}
|
| 176 |
+
],
|
| 177 |
+
"metadata": {
|
| 178 |
+
@@ -85,7 +141,7 @@
|
| 179 |
+
"name": "python",
|
| 180 |
+
"nbconvert_exporter": "python",
|
| 181 |
+
"pygments_lexer": "ipython3",
|
| 182 |
+
- "version": "3.10.18"
|
| 183 |
+
+ "version": "3.10.19"
|
| 184 |
+
}
|
| 185 |
+
},
|
| 186 |
+
"nbformat": 4,
|
| 187 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 188 |
+
index 8f2c0f1..79d2e5b 100644
|
| 189 |
+
--- a/rl_v.py
|
| 190 |
+
+++ b/rl_v.py
|
| 191 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 192 |
+
def env_prefix(env) -> str:
|
| 193 |
+
return (
|
| 194 |
+
"source ~/.bashrc && "
|
| 195 |
+
- f"source activate {env} && "
|
| 196 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 197 |
+
+ "conda init && "
|
| 198 |
+
+ f"conda activate {env} && "
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
config = get_config()
|
| 202 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 203 |
+
index 4068fc8..d284a23 100644
|
| 204 |
+
--- a/sample/llava/conversation.py
|
| 205 |
+
+++ b/sample/llava/conversation.py
|
| 206 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 207 |
+
if self.version == "llama_v3":
|
| 208 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 209 |
+
elif self.version == "llava_llada":
|
| 210 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 211 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 212 |
+
else:
|
| 213 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 214 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 215 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 216 |
+
index 3db5302..96e33b5 100644
|
| 217 |
+
--- a/sample/llava/model/builder.py
|
| 218 |
+
+++ b/sample/llava/model/builder.py
|
| 219 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 220 |
+
elif "llada" in model_name.lower():
|
| 221 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 222 |
+
|
| 223 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 224 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 225 |
+
+
|
| 226 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 227 |
+
if customized_config is None:
|
| 228 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 229 |
+
else:
|
| 230 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 231 |
+
index 4068fc8..d284a23 100644
|
| 232 |
+
--- a/train/llava/conversation.py
|
| 233 |
+
+++ b/train/llava/conversation.py
|
| 234 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 235 |
+
if self.version == "llama_v3":
|
| 236 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 237 |
+
elif self.version == "llava_llada":
|
| 238 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 239 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 240 |
+
else:
|
| 241 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 242 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 243 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 244 |
+
index 3db5302..aafab95 100644
|
| 245 |
+
--- a/train/llava/model/builder.py
|
| 246 |
+
+++ b/train/llava/model/builder.py
|
| 247 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 248 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 249 |
+
elif "llada" in model_name.lower():
|
| 250 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 251 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 252 |
+
|
| 253 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 254 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 255 |
+
if customized_config is None:
|
| 256 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 257 |
+
else:
|
| 258 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 259 |
+
index 3da28dc..8b17da3 100644
|
| 260 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 261 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 262 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 263 |
+
dpo_forward: Optional[bool] = None,
|
| 264 |
+
cache_position=None,
|
| 265 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 266 |
+
-
|
| 267 |
+
+ conversation_ids = None
|
| 268 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 269 |
+
# donate multi-dialogue
|
| 270 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..5701c74 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,62 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [],
|
| 158 |
+
+ "source": [
|
| 159 |
+
+ "import json\n",
|
| 160 |
+
+ "\n",
|
| 161 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
|
| 162 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 163 |
+
+ "save_data_list = []\n",
|
| 164 |
+
+ "save_num = 0\n",
|
| 165 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 166 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 167 |
+
+ " raw_data = json.load(fin)\n",
|
| 168 |
+
+ " for data in raw_data:\n",
|
| 169 |
+
+ " if save_num < 100:\n",
|
| 170 |
+
+ " save_data_list.append(data)\n",
|
| 171 |
+
+ " save_num += 1\n",
|
| 172 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 173 |
+
+ " print(len(save_data_list))\n"
|
| 174 |
+
+ ]
|
| 175 |
+
}
|
| 176 |
+
],
|
| 177 |
+
"metadata": {
|
| 178 |
+
@@ -85,7 +141,7 @@
|
| 179 |
+
"name": "python",
|
| 180 |
+
"nbconvert_exporter": "python",
|
| 181 |
+
"pygments_lexer": "ipython3",
|
| 182 |
+
- "version": "3.10.18"
|
| 183 |
+
+ "version": "3.10.19"
|
| 184 |
+
}
|
| 185 |
+
},
|
| 186 |
+
"nbformat": 4,
|
| 187 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 188 |
+
index 8f2c0f1..79d2e5b 100644
|
| 189 |
+
--- a/rl_v.py
|
| 190 |
+
+++ b/rl_v.py
|
| 191 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 192 |
+
def env_prefix(env) -> str:
|
| 193 |
+
return (
|
| 194 |
+
"source ~/.bashrc && "
|
| 195 |
+
- f"source activate {env} && "
|
| 196 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 197 |
+
+ "conda init && "
|
| 198 |
+
+ f"conda activate {env} && "
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
config = get_config()
|
| 202 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 203 |
+
index 4068fc8..d284a23 100644
|
| 204 |
+
--- a/sample/llava/conversation.py
|
| 205 |
+
+++ b/sample/llava/conversation.py
|
| 206 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 207 |
+
if self.version == "llama_v3":
|
| 208 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 209 |
+
elif self.version == "llava_llada":
|
| 210 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 211 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 212 |
+
else:
|
| 213 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 214 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 215 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 216 |
+
index 3db5302..96e33b5 100644
|
| 217 |
+
--- a/sample/llava/model/builder.py
|
| 218 |
+
+++ b/sample/llava/model/builder.py
|
| 219 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 220 |
+
elif "llada" in model_name.lower():
|
| 221 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 222 |
+
|
| 223 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 224 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 225 |
+
+
|
| 226 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 227 |
+
if customized_config is None:
|
| 228 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 229 |
+
else:
|
| 230 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 231 |
+
index 4068fc8..d284a23 100644
|
| 232 |
+
--- a/train/llava/conversation.py
|
| 233 |
+
+++ b/train/llava/conversation.py
|
| 234 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 235 |
+
if self.version == "llama_v3":
|
| 236 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 237 |
+
elif self.version == "llava_llada":
|
| 238 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 239 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 240 |
+
else:
|
| 241 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 242 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 243 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 244 |
+
index 3db5302..aafab95 100644
|
| 245 |
+
--- a/train/llava/model/builder.py
|
| 246 |
+
+++ b/train/llava/model/builder.py
|
| 247 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 248 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 249 |
+
elif "llada" in model_name.lower():
|
| 250 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 251 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 252 |
+
|
| 253 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 254 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 255 |
+
if customized_config is None:
|
| 256 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 257 |
+
else:
|
| 258 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 259 |
+
index 3da28dc..8b17da3 100644
|
| 260 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 261 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 262 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 263 |
+
dpo_forward: Optional[bool] = None,
|
| 264 |
+
cache_position=None,
|
| 265 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 266 |
+
-
|
| 267 |
+
+ conversation_ids = None
|
| 268 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 269 |
+
# donate multi-dialogue
|
| 270 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
wandb/run-20260322_031609-y30zeekk/files/output.log
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
03/22/2026 03:16:15 - INFO - root - Saving config to sft_lladav_video_reversal_3_21/config.yaml
|
| 2 |
+
Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/LLaDA-V
|
| 3 |
+
Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
|
| 4 |
+
Loading checkpoint shards: 100%|██████████| 6/6 [00:25<00:00, 4.19s/it]
|
| 5 |
+
Rank 0: Model Class: LlavaLLaDAModelLM
|
| 6 |
+
/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
|
| 7 |
+
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
|
| 8 |
+
warnings.warn(
|
| 9 |
+
Before initializing optimizer states
|
| 10 |
+
MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB
|
| 11 |
+
CPU Virtual Memory: used = 133.3 GB, percent = 6.6%
|
| 12 |
+
After initializing optimizer states
|
| 13 |
+
MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB
|
| 14 |
+
CPU Virtual Memory: used = 158.85 GB, percent = 7.9%
|
| 15 |
+
After initializing ZeRO optimizer
|
| 16 |
+
MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB
|
| 17 |
+
CPU Virtual Memory: used = 160.29 GB, percent = 8.0%
|
| 18 |
+
03/22/2026 03:18:02 - INFO - __main__ - ***** Running LLaDA-V SFT training *****
|
| 19 |
+
03/22/2026 03:18:02 - INFO - __main__ - Num variants = 39024
|
| 20 |
+
03/22/2026 03:18:02 - INFO - __main__ - Num training steps = 611
|
| 21 |
+
03/22/2026 03:18:02 - INFO - __main__ - Instantaneous batch size per device = 2
|
| 22 |
+
03/22/2026 03:18:02 - INFO - __main__ - Total train batch size (parallel * accum) = 64
|
| 23 |
+
03/22/2026 03:18:02 - INFO - __main__ - Gradient Accumulation steps = 4
|
| 24 |
+
Epoch 1/1: 0%| | 0/2439 [00:00<?, ?it/s]/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/checkpoint.py:87: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
|
| 25 |
+
warnings.warn(
|
| 26 |
+
Epoch 1/1: 2%|▏ | 39/2439 [05:16<7:38:40, 11.47s/it]
|
wandb/run-20260322_031609-y30zeekk/files/requirements.txt
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==4.52.4
|
| 2 |
+
nvidia-nvtx-cu12==12.4.127
|
| 3 |
+
deepspeed==0.18.6
|
| 4 |
+
typing-inspection==0.4.2
|
| 5 |
+
scipy==1.15.3
|
| 6 |
+
MarkupSafe==3.0.3
|
| 7 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 8 |
+
yarl==1.22.0
|
| 9 |
+
tornado==6.5.4
|
| 10 |
+
nvidia-curand-cu12==10.3.5.147
|
| 11 |
+
nest-asyncio==1.6.0
|
| 12 |
+
safetensors==0.7.0
|
| 13 |
+
async-timeout==5.0.1
|
| 14 |
+
typing_extensions==4.15.0
|
| 15 |
+
asttokens==3.0.1
|
| 16 |
+
cuda-python==13.1.1
|
| 17 |
+
executing==2.2.1
|
| 18 |
+
nvidia-cudnn-frontend==1.18.0
|
| 19 |
+
comm==0.2.3
|
| 20 |
+
tokenizers==0.21.4
|
| 21 |
+
six==1.17.0
|
| 22 |
+
Pygments==2.19.2
|
| 23 |
+
nvidia-cusparselt-cu12==0.6.2
|
| 24 |
+
apache-tvm-ffi==0.1.8.post2
|
| 25 |
+
flash_attn==2.7.4.post1
|
| 26 |
+
tqdm==4.67.3
|
| 27 |
+
tensorboard-data-server==0.7.2
|
| 28 |
+
annotated-types==0.7.0
|
| 29 |
+
math-verify==0.9.0
|
| 30 |
+
idna==3.11
|
| 31 |
+
regex==2026.1.15
|
| 32 |
+
nvidia-cutlass-dsl-libs-base==4.4.0
|
| 33 |
+
flashinfer-python==0.6.3
|
| 34 |
+
click==8.3.1
|
| 35 |
+
pytz==2025.2
|
| 36 |
+
wheel==0.46.3
|
| 37 |
+
ipykernel==7.2.0
|
| 38 |
+
stack-data==0.6.3
|
| 39 |
+
absl-py==2.4.0
|
| 40 |
+
wandb==0.25.0
|
| 41 |
+
pandas==2.3.3
|
| 42 |
+
termcolor==3.3.0
|
| 43 |
+
sympy==1.13.1
|
| 44 |
+
joblib==1.5.3
|
| 45 |
+
prompt_toolkit==3.0.52
|
| 46 |
+
attrs==25.4.0
|
| 47 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 48 |
+
xxhash==3.6.0
|
| 49 |
+
aiohappyeyeballs==2.6.1
|
| 50 |
+
python-dateutil==2.9.0.post0
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nvidia-ml-py==13.590.48
|
| 53 |
+
multiprocess==0.70.18
|
| 54 |
+
decorator==5.2.1
|
| 55 |
+
cuda-pathfinder==1.3.4
|
| 56 |
+
multidict==6.7.1
|
| 57 |
+
pure_eval==0.2.3
|
| 58 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 59 |
+
omegaconf==2.3.0
|
| 60 |
+
PyYAML==6.0.3
|
| 61 |
+
debugpy==1.8.20
|
| 62 |
+
psutil==7.2.2
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 65 |
+
triton==3.2.0
|
| 66 |
+
antlr4-python3-runtime==4.9.3
|
| 67 |
+
peft==0.18.1
|
| 68 |
+
aiosignal==1.4.0
|
| 69 |
+
accelerate==1.12.0
|
| 70 |
+
liger_kernel==0.7.0
|
| 71 |
+
anyio==4.12.1
|
| 72 |
+
decord==0.6.0
|
| 73 |
+
h11==0.16.0
|
| 74 |
+
urllib3==2.6.3
|
| 75 |
+
wcwidth==0.6.0
|
| 76 |
+
networkx==3.4.2
|
| 77 |
+
huggingface_hub==0.36.2
|
| 78 |
+
tzdata==2025.3
|
| 79 |
+
xformers==0.0.29.post3
|
| 80 |
+
tabulate==0.9.0
|
| 81 |
+
certifi==2026.1.4
|
| 82 |
+
tensorboard==2.20.0
|
| 83 |
+
matplotlib-inline==0.2.1
|
| 84 |
+
pip==26.0.1
|
| 85 |
+
smmap==5.0.2
|
| 86 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 87 |
+
Markdown==3.10.2
|
| 88 |
+
nvidia-nccl-cu12==2.21.5
|
| 89 |
+
aiohttp==3.13.3
|
| 90 |
+
scikit-learn==1.7.2
|
| 91 |
+
nvidia-cutlass-dsl==4.4.0
|
| 92 |
+
setuptools==82.0.0
|
| 93 |
+
jupyter_core==5.9.1
|
| 94 |
+
asyncio==4.0.0
|
| 95 |
+
einops==0.8.2
|
| 96 |
+
pydantic==2.12.5
|
| 97 |
+
av==16.1.0
|
| 98 |
+
protobuf==6.33.5
|
| 99 |
+
evaluate==0.4.6
|
| 100 |
+
Jinja2==3.1.6
|
| 101 |
+
exceptiongroup==1.3.1
|
| 102 |
+
ipython==8.38.0
|
| 103 |
+
sentry-sdk==2.53.0
|
| 104 |
+
httpx==0.28.1
|
| 105 |
+
msgpack==1.1.2
|
| 106 |
+
platformdirs==4.9.2
|
| 107 |
+
hjson==3.1.0
|
| 108 |
+
torchvision==0.21.0
|
| 109 |
+
parso==0.8.6
|
| 110 |
+
cuda-bindings==13.1.1
|
| 111 |
+
pexpect==4.9.0
|
| 112 |
+
threadpoolctl==3.6.0
|
| 113 |
+
pyarrow==23.0.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 116 |
+
Werkzeug==3.1.5
|
| 117 |
+
fsspec==2025.10.0
|
| 118 |
+
pydantic_core==2.41.5
|
| 119 |
+
GitPython==3.1.46
|
| 120 |
+
propcache==0.4.1
|
| 121 |
+
mpmath==1.3.0
|
| 122 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 123 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 124 |
+
hf-xet==1.2.0
|
| 125 |
+
charset-normalizer==3.4.4
|
| 126 |
+
dill==0.4.0
|
| 127 |
+
datasets==4.5.0
|
| 128 |
+
py-cpuinfo==9.0.0
|
| 129 |
+
sentencepiece==0.2.1
|
| 130 |
+
latex2sympy2_extended==1.11.0
|
| 131 |
+
jupyter_client==8.8.0
|
| 132 |
+
pyzmq==27.1.0
|
| 133 |
+
numpy==2.2.6
|
| 134 |
+
gitdb==4.0.12
|
| 135 |
+
grpcio==1.78.0
|
| 136 |
+
packaging==26.0
|
| 137 |
+
requests==2.32.5
|
| 138 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 139 |
+
httpcore==1.0.9
|
| 140 |
+
traitlets==5.14.3
|
| 141 |
+
filelock==3.24.2
|
| 142 |
+
ptyprocess==0.7.0
|
| 143 |
+
pillow==12.1.1
|
| 144 |
+
torch==2.6.0
|
| 145 |
+
more-itertools==10.8.0
|
| 146 |
+
wheel==0.46.3
|
| 147 |
+
jaraco.text==4.0.0
|
| 148 |
+
zipp==3.23.0
|
| 149 |
+
autocommand==2.2.2
|
| 150 |
+
jaraco.context==6.1.0
|
| 151 |
+
jaraco.functools==4.4.0
|
| 152 |
+
tomli==2.4.0
|
| 153 |
+
backports.tarfile==1.2.0
|
| 154 |
+
importlib_metadata==8.7.1
|
| 155 |
+
platformdirs==4.4.0
|
| 156 |
+
packaging==26.0
|
wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-03-21T19:16:09.677405Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"config=configs/sft_lladav_video_reversal.yaml"
|
| 7 |
+
],
|
| 8 |
+
"program": "/data4/runqing/dLLM-RL/train/sft_lladav_video_reversal.py",
|
| 9 |
+
"codePath": "train/sft_lladav_video_reversal.py",
|
| 10 |
+
"codePathLocal": "train/sft_lladav_video_reversal.py",
|
| 11 |
+
"git": {
|
| 12 |
+
"remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
|
| 13 |
+
"commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
|
| 14 |
+
},
|
| 15 |
+
"email": "shixi_runqing@staff.weibo.com",
|
| 16 |
+
"root": "/data4/runqing/dLLM-RL",
|
| 17 |
+
"host": "b1301b38c75f",
|
| 18 |
+
"executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
|
| 19 |
+
"cpu_count": 96,
|
| 20 |
+
"cpu_count_logical": 192,
|
| 21 |
+
"gpu": "NVIDIA H800",
|
| 22 |
+
"gpu_count": 8,
|
| 23 |
+
"disk": {
|
| 24 |
+
"/": {
|
| 25 |
+
"total": "806578724864",
|
| 26 |
+
"used": "571700989952"
|
| 27 |
+
}
|
| 28 |
+
},
|
| 29 |
+
"memory": {
|
| 30 |
+
"total": "2164113813504"
|
| 31 |
+
},
|
| 32 |
+
"gpu_nvidia": [
|
| 33 |
+
{
|
| 34 |
+
"name": "NVIDIA H800",
|
| 35 |
+
"memoryTotal": "85520809984",
|
| 36 |
+
"cudaCores": 16896,
|
| 37 |
+
"architecture": "Hopper",
|
| 38 |
+
"uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"name": "NVIDIA H800",
|
| 42 |
+
"memoryTotal": "85520809984",
|
| 43 |
+
"cudaCores": 16896,
|
| 44 |
+
"architecture": "Hopper",
|
| 45 |
+
"uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"name": "NVIDIA H800",
|
| 49 |
+
"memoryTotal": "85520809984",
|
| 50 |
+
"cudaCores": 16896,
|
| 51 |
+
"architecture": "Hopper",
|
| 52 |
+
"uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"name": "NVIDIA H800",
|
| 56 |
+
"memoryTotal": "85520809984",
|
| 57 |
+
"cudaCores": 16896,
|
| 58 |
+
"architecture": "Hopper",
|
| 59 |
+
"uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "NVIDIA H800",
|
| 63 |
+
"memoryTotal": "85520809984",
|
| 64 |
+
"cudaCores": 16896,
|
| 65 |
+
"architecture": "Hopper",
|
| 66 |
+
"uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"name": "NVIDIA H800",
|
| 70 |
+
"memoryTotal": "85520809984",
|
| 71 |
+
"cudaCores": 16896,
|
| 72 |
+
"architecture": "Hopper",
|
| 73 |
+
"uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"name": "NVIDIA H800",
|
| 77 |
+
"memoryTotal": "85520809984",
|
| 78 |
+
"cudaCores": 16896,
|
| 79 |
+
"architecture": "Hopper",
|
| 80 |
+
"uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "NVIDIA H800",
|
| 84 |
+
"memoryTotal": "85520809984",
|
| 85 |
+
"cudaCores": 16896,
|
| 86 |
+
"architecture": "Hopper",
|
| 87 |
+
"uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
"cudaVersion": "12.4",
|
| 91 |
+
"writerId": "0nbl2szyao6zltkyl1iimb0wrhiws85w"
|
| 92 |
+
}
|
wandb/run-20260322_031609-y30zeekk/logs/debug-core.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-22T03:16:09.743285688+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp2a9va2fu/port-917982.txt","pid":917982,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-03-22T03:16:09.744267653+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":917982}
|
| 3 |
+
{"time":"2026-03-22T03:16:09.744257174+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-917982-918484-3562169965/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-03-22T03:16:09.925032752+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-03-22T03:16:09.931813709+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"y30zeekk","id":"1(@)"}
|
| 6 |
+
{"time":"2026-03-22T03:16:14.939482566+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"y30zeekk","id":"1(@)"}
|
| 7 |
+
{"time":"2026-03-22T03:16:20.158587842+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"txikwagb0eg1"}
|
wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-22T03:16:09.932043851+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-03-22T03:16:14.939035914+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
|
| 3 |
+
{"time":"2026-03-22T03:16:14.939210648+08:00","level":"INFO","msg":"stream: created new stream","id":"y30zeekk"}
|
| 4 |
+
{"time":"2026-03-22T03:16:14.939296134+08:00","level":"INFO","msg":"handler: started","stream_id":"y30zeekk"}
|
| 5 |
+
{"time":"2026-03-22T03:16:14.939467726+08:00","level":"INFO","msg":"stream: started","id":"y30zeekk"}
|
| 6 |
+
{"time":"2026-03-22T03:16:14.939512912+08:00","level":"INFO","msg":"sender: started","stream_id":"y30zeekk"}
|
| 7 |
+
{"time":"2026-03-22T03:16:14.939515245+08:00","level":"INFO","msg":"writer: started","stream_id":"y30zeekk"}
|
| 8 |
+
{"time":"2026-03-22T03:16:14.942265397+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
|
| 9 |
+
{"time":"2026-03-22T03:16:20.122143616+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=543a564ab27a29e0506e5496b121bfa5ec3df952fb270dc6cbc8e792933f1a0d"}
|
| 10 |
+
{"time":"2026-03-22T03:16:20.122295801+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch</Resource><RequestId>189EF14496B0C686</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=543a564ab27a29e0506e5496b121bfa5ec3df952fb270dc6cbc8e792933f1a0d, Size: 11427}"}
|
| 11 |
+
{"time":"2026-03-22T03:16:20.123748872+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=65fe9536dca78a54fcb1be38cc2071ac77844ac78ae053d4f2dda5671108cda7"}
|
| 12 |
+
{"time":"2026-03-22T03:16:20.123818956+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt</Resource><RequestId>189EF14496BBA833</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=65fe9536dca78a54fcb1be38cc2071ac77844ac78ae053d4f2dda5671108cda7, Size: 2966}"}
|
| 13 |
+
{"time":"2026-03-22T03:16:20.124069975+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=76db83e542423b361e9411c108e0835fa100fe9691efcbc1f98e5e3a28fedecf"}
|
| 14 |
+
{"time":"2026-03-22T03:16:20.124180799+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json</Resource><RequestId>189EF14496AFDED4</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=76db83e542423b361e9411c108e0835fa100fe9691efcbc1f98e5e3a28fedecf, Size: 2650}"}
|
| 15 |
+
{"time":"2026-03-22T03:16:20.126422985+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8f5ee1f91644676947fa6db83d772470d8ed9fdcf739aa7e905ac62cb16f53d"}
|
| 16 |
+
{"time":"2026-03-22T03:16:20.126566466+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189EF14496C34324</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8f5ee1f91644676947fa6db83d772470d8ed9fdcf739aa7e905ac62cb16f53d, Size: 11427}"}
|
| 17 |
+
{"time":"2026-03-22T03:16:20.13421156+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59e9f74c84de8a143e6c6b9f2aba14af539692b331769d7512155279863bd62c"}
|
| 18 |
+
{"time":"2026-03-22T03:16:20.134366382+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py</Resource><RequestId>189EF14496CC1BA5</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py, Name: code/train/sft_lladav_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59e9f74c84de8a143e6c6b9f2aba14af539692b331769d7512155279863bd62c, Size: 29865}"}
|
wandb/run-20260322_031609-y30zeekk/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
|
| 2 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Configure stats pid to 917982
|
| 3 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/logs/debug.log
|
| 5 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log
|
| 6 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {'code_path': 'code/train/sft_lladav_video_reversal.py'}}
|
| 9 |
+
2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-22 03:16:09,925 INFO MainThread:917982 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-22 03:16:09,928 INFO MainThread:917982 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-22 03:16:09,929 INFO MainThread:917982 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-22 03:16:09,935 INFO MainThread:917982 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-22 03:16:15,046 INFO MainThread:917982 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-22 03:16:15,157 INFO MainThread:917982 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-22 03:16:15,159 INFO MainThread:917982 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'y30zeekk', 'experiment.project': 'sft_lladav_video_reversal_3_21', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lladav_video_reversal_3_21/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train_cleaned', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 4, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.1, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lladav_video_reversal.yaml'}
|
wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb
ADDED
|
Binary file (98.3 kB). View file
|
|
|
wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py
ADDED
|
@@ -0,0 +1,806 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 4 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
| 5 |
+
from datetime import timedelta
|
| 6 |
+
import json
|
| 7 |
+
import logging
|
| 8 |
+
import math
|
| 9 |
+
import shutil
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Dict, List, Any
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from torch.optim import AdamW
|
| 18 |
+
from torch.utils.data import Dataset, DataLoader
|
| 19 |
+
|
| 20 |
+
from omegaconf import OmegaConf
|
| 21 |
+
import wandb
|
| 22 |
+
|
| 23 |
+
from accelerate import Accelerator
|
| 24 |
+
from accelerate.logging import get_logger
|
| 25 |
+
from accelerate.utils import set_seed
|
| 26 |
+
|
| 27 |
+
from train.utils import get_config, flatten_omega_conf, AverageMeter
|
| 28 |
+
from models.lr_schedulers import get_scheduler
|
| 29 |
+
from models.logging import set_verbosity_info, set_verbosity_error
|
| 30 |
+
|
| 31 |
+
from transformers.utils import logging as hf_logging
|
| 32 |
+
hf_logging.set_verbosity_error()
|
| 33 |
+
|
| 34 |
+
from llava.model.builder import load_pretrained_model
|
| 35 |
+
from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord
|
| 36 |
+
from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
|
| 37 |
+
from llava.conversation import conv_templates
|
| 38 |
+
|
| 39 |
+
from termcolor import cprint
|
| 40 |
+
|
| 41 |
+
logger = get_logger(__name__, log_level="INFO")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def collapse_k_unique(lst, k: int):
|
| 45 |
+
if k <= 0:
|
| 46 |
+
raise ValueError("k must be > 0")
|
| 47 |
+
uniq = sorted(set(lst))
|
| 48 |
+
mapping = {}
|
| 49 |
+
n = len(uniq)
|
| 50 |
+
for idx, val in enumerate(uniq):
|
| 51 |
+
group = idx // k
|
| 52 |
+
end_idx = min((group + 1) * k - 1, n - 1)
|
| 53 |
+
rep = uniq[end_idx]
|
| 54 |
+
mapping[val] = rep
|
| 55 |
+
return [mapping[x] for x in lst]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
|
| 59 |
+
import copy
|
| 60 |
+
conv = copy.deepcopy(conv_templates["llava_llada"])
|
| 61 |
+
conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
|
| 62 |
+
conv.append_message(conv.roles[1], response_text)
|
| 63 |
+
return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
|
| 67 |
+
conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
|
| 68 |
+
ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
|
| 69 |
+
return ids.squeeze(0)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
|
| 73 |
+
from PIL import Image
|
| 74 |
+
from types import SimpleNamespace
|
| 75 |
+
|
| 76 |
+
if isinstance(model_cfg, dict):
|
| 77 |
+
model_cfg = SimpleNamespace(**model_cfg)
|
| 78 |
+
|
| 79 |
+
if media_type == "image":
|
| 80 |
+
image = Image.open(abs_path).convert("RGB")
|
| 81 |
+
if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
|
| 82 |
+
setattr(model_cfg, "image_aspect_ratio", "pad")
|
| 83 |
+
if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
|
| 84 |
+
setattr(model_cfg, "mm_patch_merge_type", "flat")
|
| 85 |
+
|
| 86 |
+
image_tensor_list = process_images([image], image_processor, model_cfg)
|
| 87 |
+
image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
|
| 88 |
+
image_sizes = [image.size]
|
| 89 |
+
return image_tensor_list, image_sizes, None
|
| 90 |
+
|
| 91 |
+
elif media_type == "video":
|
| 92 |
+
if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
|
| 93 |
+
if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 0)
|
| 94 |
+
if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", False)
|
| 95 |
+
|
| 96 |
+
video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
|
| 97 |
+
image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
|
| 98 |
+
image_tensor = image_tensor.to(dtype=dtype, device=device)
|
| 99 |
+
|
| 100 |
+
if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
|
| 101 |
+
v_size = (video_frames.shape[2], video_frames.shape[1])
|
| 102 |
+
else:
|
| 103 |
+
v_size = (0, 0)
|
| 104 |
+
|
| 105 |
+
return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
|
| 109 |
+
out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
|
| 110 |
+
out[-x.numel():] = x
|
| 111 |
+
return out
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
|
| 115 |
+
if x.size(0) == L:
|
| 116 |
+
return x
|
| 117 |
+
out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
|
| 118 |
+
out[-x.size(0):] = x
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
|
| 123 |
+
p = sample.get(f"{media_type}_abs_path")
|
| 124 |
+
if p is None:
|
| 125 |
+
p = sample.get(media_type) or sample.get(f"{media_type}_path")
|
| 126 |
+
if p is None:
|
| 127 |
+
return None
|
| 128 |
+
if not os.path.isabs(p) and dataset_root:
|
| 129 |
+
p = os.path.join(dataset_root, p)
|
| 130 |
+
try:
|
| 131 |
+
return os.path.abspath(p)
|
| 132 |
+
except Exception:
|
| 133 |
+
return p
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class RLVariantDatasetLLADAV(Dataset):
|
| 137 |
+
def __init__(self, samples: List[dict], config):
|
| 138 |
+
super().__init__()
|
| 139 |
+
self.samples = samples
|
| 140 |
+
self.config = config
|
| 141 |
+
self.variants: List[dict] = []
|
| 142 |
+
|
| 143 |
+
# logp_old_tok mapping: idx -> tensor
|
| 144 |
+
self.logp_old_tok: Dict[int, torch.Tensor] = {}
|
| 145 |
+
|
| 146 |
+
self._build_variants()
|
| 147 |
+
|
| 148 |
+
def _build_variants(self):
|
| 149 |
+
sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
|
| 150 |
+
question_prompt = "\nOnly give the best option."
|
| 151 |
+
multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
|
| 152 |
+
|
| 153 |
+
rng = np.random.default_rng(seed=self.config.training.seed or 42)
|
| 154 |
+
idx_counter = 0
|
| 155 |
+
|
| 156 |
+
for sid, s in enumerate(self.samples):
|
| 157 |
+
# Rollout produces "extracted_output" or "response", fall back if needed
|
| 158 |
+
response_text = s.get("response", s.get("extracted_output", ""))
|
| 159 |
+
if isinstance(response_text, list) and len(response_text) > 0:
|
| 160 |
+
response_text = response_text[0] # Just in case it's nested
|
| 161 |
+
|
| 162 |
+
# If options exist (video multiple choice), construct full prompt
|
| 163 |
+
if "options" in s:
|
| 164 |
+
prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt
|
| 165 |
+
else:
|
| 166 |
+
prompt_text = s["prompt"]
|
| 167 |
+
|
| 168 |
+
step_map = s.get("step_map", [])
|
| 169 |
+
|
| 170 |
+
if self.config.training.method == "TraceRL":
|
| 171 |
+
order_list = collapse_k_unique(step_map, self.config.training.shrink)
|
| 172 |
+
uniq_steps = sorted(set(order_list))
|
| 173 |
+
for st in uniq_steps:
|
| 174 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None))
|
| 175 |
+
idx_counter += 1
|
| 176 |
+
|
| 177 |
+
elif self.config.training.method == "random_masking":
|
| 178 |
+
for _ in range(self.config.training.mask_times_per_sample):
|
| 179 |
+
t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
|
| 180 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 181 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed))
|
| 182 |
+
idx_counter += 1
|
| 183 |
+
|
| 184 |
+
elif self.config.training.method == "coupled":
|
| 185 |
+
for _ in range(self.config.training.mask_times_per_sample):
|
| 186 |
+
t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
|
| 187 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 188 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed))
|
| 189 |
+
idx_counter += 1
|
| 190 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed))
|
| 191 |
+
idx_counter += 1
|
| 192 |
+
else:
|
| 193 |
+
raise ValueError(f"Unknown RL method: {self.config.training.method}")
|
| 194 |
+
|
| 195 |
+
def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed):
|
| 196 |
+
return dict(
|
| 197 |
+
idx=idx,
|
| 198 |
+
sample_id=sid,
|
| 199 |
+
prompt=prompt,
|
| 200 |
+
response=response,
|
| 201 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 202 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 203 |
+
reward=s.get("reward", 0.0),
|
| 204 |
+
var_type=var_type,
|
| 205 |
+
var_param=var_param,
|
| 206 |
+
step_map=step_map,
|
| 207 |
+
mask_seed=mask_seed,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
def __len__(self):
|
| 211 |
+
return len(self.variants)
|
| 212 |
+
|
| 213 |
+
def __getitem__(self, idx):
|
| 214 |
+
return self.variants[idx]
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
|
| 218 |
+
out: Dict[str, List[Any]] = {}
|
| 219 |
+
keys = batch[0].keys()
|
| 220 |
+
for k in keys:
|
| 221 |
+
out[k] = [b[k] for b in batch]
|
| 222 |
+
return out
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
@torch.no_grad()
|
| 226 |
+
def prepare_batch_noisy_embeds_rl(
|
| 227 |
+
accelerator,
|
| 228 |
+
model,
|
| 229 |
+
tokenizer,
|
| 230 |
+
image_processor,
|
| 231 |
+
batch: Dict[str, List[Any]],
|
| 232 |
+
mask_id: int,
|
| 233 |
+
dataset_media_root: str | None,
|
| 234 |
+
) -> tuple | None:
|
| 235 |
+
|
| 236 |
+
device = accelerator.device
|
| 237 |
+
bsz = len(batch["idx"])
|
| 238 |
+
|
| 239 |
+
group: Dict[int, List[int]] = {}
|
| 240 |
+
for i, sid in enumerate(batch["sample_id"]):
|
| 241 |
+
group.setdefault(int(sid), []).append(i)
|
| 242 |
+
|
| 243 |
+
sample_cache: Dict[int, Dict[str, Any]] = {}
|
| 244 |
+
for sid, pos_list in group.items():
|
| 245 |
+
first_pos = pos_list[0]
|
| 246 |
+
prompt_text = batch["prompt"][first_pos]
|
| 247 |
+
response_text = batch["response"][first_pos]
|
| 248 |
+
|
| 249 |
+
sample_stub = {
|
| 250 |
+
"image_abs_path": batch.get("image_abs_path", [None])[first_pos],
|
| 251 |
+
"video_abs_path": batch.get("video_abs_path", [None])[first_pos],
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
media_type = None
|
| 255 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
|
| 256 |
+
if p_abs is not None:
|
| 257 |
+
media_type = "video"
|
| 258 |
+
else:
|
| 259 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
|
| 260 |
+
if p_abs is not None:
|
| 261 |
+
media_type = "image"
|
| 262 |
+
|
| 263 |
+
img_list, image_sizes, modalities = None, None, []
|
| 264 |
+
if p_abs is not None:
|
| 265 |
+
img_list, image_sizes, video_stats = load_and_process_media(
|
| 266 |
+
p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
|
| 267 |
+
)
|
| 268 |
+
modalities = [media_type]
|
| 269 |
+
|
| 270 |
+
if media_type == "video" and getattr(model.config, "add_time_instruction", False):
|
| 271 |
+
video_time, frame_time, num_frames = video_stats
|
| 272 |
+
time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
|
| 273 |
+
clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
|
| 274 |
+
prompt_text = f"{time_instruction}\n{clean_prompt}"
|
| 275 |
+
|
| 276 |
+
input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
|
| 277 |
+
end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
|
| 278 |
+
seq_len = len(end_header_seq)
|
| 279 |
+
pos_eh = -1
|
| 280 |
+
if seq_len > 0 and input_ids_text.numel() >= seq_len:
|
| 281 |
+
ids_list = input_ids_text.tolist()
|
| 282 |
+
for i in range(len(ids_list) - seq_len, -1, -1):
|
| 283 |
+
if ids_list[i:i + seq_len] == end_header_seq:
|
| 284 |
+
pos_eh = i + seq_len - 1
|
| 285 |
+
break
|
| 286 |
+
assert pos_eh != -1, " <|end_header_id|> not found"
|
| 287 |
+
start_pos = pos_eh + 1
|
| 288 |
+
|
| 289 |
+
labels_text = input_ids_text.clone()
|
| 290 |
+
labels_text[:start_pos] = -100
|
| 291 |
+
|
| 292 |
+
(_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
|
| 293 |
+
input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
|
| 294 |
+
)
|
| 295 |
+
clean_embeds = clean_embeds.squeeze(0)
|
| 296 |
+
new_labels = new_labels.squeeze(0)
|
| 297 |
+
|
| 298 |
+
eot_id = 126348
|
| 299 |
+
eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
|
| 300 |
+
if eot_indices.numel() > 0:
|
| 301 |
+
last_eot_pos = eot_indices[-1, 0]
|
| 302 |
+
new_labels[last_eot_pos:] = -100
|
| 303 |
+
asst_mask = (new_labels != -100)
|
| 304 |
+
asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1)
|
| 305 |
+
|
| 306 |
+
sample_cache[sid] = dict(
|
| 307 |
+
clean_embeds=clean_embeds,
|
| 308 |
+
new_labels=new_labels,
|
| 309 |
+
asst_idx=asst_idx,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)]
|
| 313 |
+
Lmax = max(lengths)
|
| 314 |
+
mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
|
| 315 |
+
|
| 316 |
+
noisy_list, labels_list, p_mask_list, attn_list = [], [], [], []
|
| 317 |
+
reward_list, valid_idxs = [], []
|
| 318 |
+
|
| 319 |
+
for i in range(bsz):
|
| 320 |
+
sid = int(batch["sample_id"][i])
|
| 321 |
+
var_type = batch["var_type"][i]
|
| 322 |
+
var_param = batch["var_param"][i]
|
| 323 |
+
|
| 324 |
+
clean_embeds = sample_cache[sid]["clean_embeds"].clone()
|
| 325 |
+
new_labels = sample_cache[sid]["new_labels"].clone()
|
| 326 |
+
asst_idx = sample_cache[sid]["asst_idx"]
|
| 327 |
+
|
| 328 |
+
L = clean_embeds.size(0)
|
| 329 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 330 |
+
M = asst_idx.numel()
|
| 331 |
+
|
| 332 |
+
if M == 0:
|
| 333 |
+
continue # skip badly formatted inputs
|
| 334 |
+
|
| 335 |
+
if var_type == "TraceRL":
|
| 336 |
+
step_map = batch["step_map"][i]
|
| 337 |
+
order = torch.as_tensor(step_map, device=device)
|
| 338 |
+
m_len = min(M, order.numel())
|
| 339 |
+
order_full = torch.full((L,), -1, dtype=torch.long, device=device)
|
| 340 |
+
order_full[asst_idx[:m_len]] = order[:m_len]
|
| 341 |
+
step_val = int(var_param)
|
| 342 |
+
|
| 343 |
+
tgt_mask = (order_full == step_val)
|
| 344 |
+
pmask = tgt_mask
|
| 345 |
+
mask_pos = (order_full >= step_val)
|
| 346 |
+
|
| 347 |
+
noisy_embeds = clean_embeds.clone()
|
| 348 |
+
noisy_embeds[mask_pos] = mask_embed
|
| 349 |
+
|
| 350 |
+
elif var_type == "random_masking" or var_type.startswith("coupled"):
|
| 351 |
+
if M == 0:
|
| 352 |
+
# M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward
|
| 353 |
+
noisy_embeds = clean_embeds.clone()
|
| 354 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 355 |
+
else:
|
| 356 |
+
t = float(var_param)
|
| 357 |
+
# 更安全的 seed 获取方式
|
| 358 |
+
mask_seed = batch.get("mask_seed", [None] * bsz)[i]
|
| 359 |
+
if mask_seed is None:
|
| 360 |
+
mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
|
| 361 |
+
|
| 362 |
+
gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
|
| 363 |
+
rnd_cpu = torch.rand(M, generator=gen) < t
|
| 364 |
+
|
| 365 |
+
# 如果是 coupled_comp (对比样本),需要取反
|
| 366 |
+
if var_type == "coupled_comp":
|
| 367 |
+
rnd_cpu = ~rnd_cpu
|
| 368 |
+
|
| 369 |
+
rnd = rnd_cpu.to(device)
|
| 370 |
+
|
| 371 |
+
# 🌟 上帝之手:强行兜底逻辑
|
| 372 |
+
# 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁
|
| 373 |
+
if not rnd.any():
|
| 374 |
+
# 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反,
|
| 375 |
+
# 除非 coupled_rand 全是 True,否则 comp 不可能全 False。
|
| 376 |
+
# 这里统一下一个兜底,非常安全。
|
| 377 |
+
rand_pos = torch.randint(0, M, (1,), device=device)
|
| 378 |
+
rnd[rand_pos] = True
|
| 379 |
+
|
| 380 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 381 |
+
pmask[asst_idx] = rnd
|
| 382 |
+
|
| 383 |
+
noisy_embeds = clean_embeds.clone()
|
| 384 |
+
noisy_embeds[pmask] = mask_embed
|
| 385 |
+
|
| 386 |
+
# elif var_type == "random_masking" or var_type.startswith("coupled"):
|
| 387 |
+
# t = float(var_param)
|
| 388 |
+
# mask_seed = batch["mask_seed"][i]
|
| 389 |
+
# gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
|
| 390 |
+
# rnd_cpu = torch.rand(M, generator=gen) < t
|
| 391 |
+
|
| 392 |
+
# if var_type == "coupled_comp":
|
| 393 |
+
# rnd_cpu = ~rnd_cpu
|
| 394 |
+
|
| 395 |
+
# pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 396 |
+
# pmask[asst_idx] = rnd_cpu.to(device)
|
| 397 |
+
|
| 398 |
+
# noisy_embeds = clean_embeds.clone()
|
| 399 |
+
# noisy_embeds[pmask] = mask_embed
|
| 400 |
+
|
| 401 |
+
if not pmask.any():
|
| 402 |
+
print('************not pmask.any()**********')
|
| 403 |
+
continue
|
| 404 |
+
|
| 405 |
+
noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
|
| 406 |
+
labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
|
| 407 |
+
p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
|
| 408 |
+
attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
|
| 409 |
+
attn[-L:] = True
|
| 410 |
+
attn_list.append(attn)
|
| 411 |
+
reward_list.append(float(batch["reward"][i]))
|
| 412 |
+
valid_idxs.append(batch["idx"][i])
|
| 413 |
+
|
| 414 |
+
if len(noisy_list) == 0:
|
| 415 |
+
return None
|
| 416 |
+
|
| 417 |
+
noisy_batch = torch.stack(noisy_list, dim=0)
|
| 418 |
+
labels_batch = torch.stack(labels_list, dim=0)
|
| 419 |
+
p_mask_batch = torch.stack(p_mask_list, dim=0)
|
| 420 |
+
attn_batch = torch.stack(attn_list, dim=0)
|
| 421 |
+
reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32)
|
| 422 |
+
|
| 423 |
+
return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
@torch.no_grad()
|
| 427 |
+
def compute_logp_old_tok_parallel(
|
| 428 |
+
accelerator,
|
| 429 |
+
model,
|
| 430 |
+
tokenizer,
|
| 431 |
+
image_processor,
|
| 432 |
+
dataset,
|
| 433 |
+
dl,
|
| 434 |
+
mask_id,
|
| 435 |
+
config):
|
| 436 |
+
|
| 437 |
+
model.eval()
|
| 438 |
+
logger.info("Computing reference log probabilities (old_lp) for all variants...")
|
| 439 |
+
|
| 440 |
+
for batch in dl:
|
| 441 |
+
out = prepare_batch_noisy_embeds_rl(
|
| 442 |
+
accelerator=accelerator,
|
| 443 |
+
model=model,
|
| 444 |
+
tokenizer=tokenizer,
|
| 445 |
+
image_processor=image_processor,
|
| 446 |
+
batch=batch,
|
| 447 |
+
mask_id=mask_id,
|
| 448 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 449 |
+
)
|
| 450 |
+
if out is None:
|
| 451 |
+
continue
|
| 452 |
+
|
| 453 |
+
noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out
|
| 454 |
+
|
| 455 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True)
|
| 456 |
+
logits = model.lm_head(outputs.last_hidden_state).float()
|
| 457 |
+
|
| 458 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 459 |
+
safe_labels = labels_batch.clone()
|
| 460 |
+
safe_labels[labels_batch == -100] = 0
|
| 461 |
+
tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
|
| 462 |
+
|
| 463 |
+
for i, idx in enumerate(valid_idxs):
|
| 464 |
+
dataset.logp_old_tok[idx] = tok_lp[i].float().cpu()
|
| 465 |
+
|
| 466 |
+
accelerator.wait_for_everyone()
|
| 467 |
+
model.train()
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
def save_checkpoint(model, tokenizer, config, accelerator, name):
|
| 471 |
+
output_dir = Path(config.experiment.project)
|
| 472 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 473 |
+
|
| 474 |
+
checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
|
| 475 |
+
|
| 476 |
+
if accelerator.is_main_process and checkpoints_total_limit is not None:
|
| 477 |
+
ckpts = sorted(
|
| 478 |
+
[d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
|
| 479 |
+
key=lambda p: int(p.name.split("-")[1]),
|
| 480 |
+
)
|
| 481 |
+
if len(ckpts) >= checkpoints_total_limit:
|
| 482 |
+
to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
|
| 483 |
+
logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
|
| 484 |
+
for p in to_remove:
|
| 485 |
+
shutil.rmtree(p, ignore_errors=True)
|
| 486 |
+
|
| 487 |
+
save_base = output_dir / "ckpt"
|
| 488 |
+
save_base.mkdir(exist_ok=True)
|
| 489 |
+
|
| 490 |
+
model_to_save = accelerator.unwrap_model(model)
|
| 491 |
+
state_dict = accelerator.get_state_dict(model)
|
| 492 |
+
|
| 493 |
+
if accelerator.is_main_process:
|
| 494 |
+
model_to_save.save_pretrained(
|
| 495 |
+
save_base / name,
|
| 496 |
+
save_function=accelerator.save,
|
| 497 |
+
state_dict=state_dict,
|
| 498 |
+
safe_serialization=True,
|
| 499 |
+
)
|
| 500 |
+
tokenizer.save_pretrained(str(save_base / name))
|
| 501 |
+
|
| 502 |
+
metadata = {
|
| 503 |
+
"save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 504 |
+
}
|
| 505 |
+
with (save_base / "metadata.json").open("w") as f:
|
| 506 |
+
json.dump(metadata, f, indent=2)
|
| 507 |
+
|
| 508 |
+
logger.info(f"Saved model + tokenizer to {save_base / name}")
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def main():
|
| 512 |
+
config = get_config()
|
| 513 |
+
|
| 514 |
+
project_name = config.experiment.project
|
| 515 |
+
if config.experiment.current_epoch == 1:
|
| 516 |
+
pretrained_model = config.model.pretrained_model
|
| 517 |
+
else:
|
| 518 |
+
pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name
|
| 519 |
+
|
| 520 |
+
if config.training.enable_tf32:
|
| 521 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 522 |
+
torch.backends.cudnn.benchmark = True
|
| 523 |
+
torch.backends.cudnn.deterministic = False
|
| 524 |
+
|
| 525 |
+
config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
|
| 526 |
+
accelerator = Accelerator(
|
| 527 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 528 |
+
mixed_precision=config.training.mixed_precision,
|
| 529 |
+
log_with='wandb',
|
| 530 |
+
project_dir=config.experiment.logging_dir,
|
| 531 |
+
split_batches=True,
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
logging.basicConfig(
|
| 535 |
+
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
| 536 |
+
datefmt="%m/%d/%Y %H:%M:%S",
|
| 537 |
+
level=logging.INFO,
|
| 538 |
+
)
|
| 539 |
+
logger.info(accelerator.state, main_process_only=False)
|
| 540 |
+
if accelerator.is_local_main_process:
|
| 541 |
+
set_verbosity_info()
|
| 542 |
+
else:
|
| 543 |
+
set_verbosity_error()
|
| 544 |
+
|
| 545 |
+
if accelerator.is_main_process:
|
| 546 |
+
resume_wandb_run = config.wandb.resume
|
| 547 |
+
run_id = config.wandb.get("run_id", None)
|
| 548 |
+
if run_id is None:
|
| 549 |
+
resume_wandb_run = False
|
| 550 |
+
run_id = wandb.util.generate_id()
|
| 551 |
+
config.wandb.run_id = run_id
|
| 552 |
+
|
| 553 |
+
wandb_init_kwargs = dict(
|
| 554 |
+
name=config.experiment.project,
|
| 555 |
+
id=run_id,
|
| 556 |
+
resume=resume_wandb_run,
|
| 557 |
+
entity=config.wandb.get("entity", None),
|
| 558 |
+
config_exclude_keys=[],
|
| 559 |
+
)
|
| 560 |
+
wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
|
| 561 |
+
wandb_config.pop("experiment.resume_from_checkpoint", None)
|
| 562 |
+
|
| 563 |
+
accelerator.init_trackers(
|
| 564 |
+
config.experiment.project,
|
| 565 |
+
config=wandb_config,
|
| 566 |
+
init_kwargs={"wandb": wandb_init_kwargs},
|
| 567 |
+
)
|
| 568 |
+
|
| 569 |
+
if accelerator.is_main_process:
|
| 570 |
+
os.makedirs(config.experiment.project, exist_ok=True)
|
| 571 |
+
config_path = Path(config.experiment.project) / "config.yaml"
|
| 572 |
+
logging.info(f"Saving config to {config_path}")
|
| 573 |
+
OmegaConf.save(config, config_path)
|
| 574 |
+
|
| 575 |
+
if config.training.seed is not None:
|
| 576 |
+
set_seed(config.training.seed)
|
| 577 |
+
|
| 578 |
+
logger.info("Loading LLaDA-V model and tokenizer")
|
| 579 |
+
|
| 580 |
+
tokenizer, model, image_processor, _ = load_pretrained_model(
|
| 581 |
+
pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
|
| 582 |
+
)
|
| 583 |
+
model = model.to(accelerator.device)
|
| 584 |
+
|
| 585 |
+
special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
|
| 586 |
+
num_new = tokenizer.add_special_tokens(special_tokens)
|
| 587 |
+
if num_new > 0:
|
| 588 |
+
model.resize_token_embeddings(len(tokenizer))
|
| 589 |
+
if tokenizer.pad_token is None:
|
| 590 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 591 |
+
|
| 592 |
+
if getattr(config.training, "gradient_checkpointing_enable", False):
|
| 593 |
+
try:
|
| 594 |
+
model.enable_input_require_grads()
|
| 595 |
+
except Exception:
|
| 596 |
+
pass
|
| 597 |
+
try:
|
| 598 |
+
model.gradient_checkpointing_enable()
|
| 599 |
+
except Exception:
|
| 600 |
+
pass
|
| 601 |
+
|
| 602 |
+
mask_id = 126336
|
| 603 |
+
|
| 604 |
+
optimizer_config = config.optimizer.params
|
| 605 |
+
no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
|
| 606 |
+
optimizer_grouped_parameters = [
|
| 607 |
+
{
|
| 608 |
+
"params": [p for n, p in model.named_parameters() if
|
| 609 |
+
p.requires_grad and not any(nd in n for nd in no_decay)],
|
| 610 |
+
"weight_decay": optimizer_config.weight_decay,
|
| 611 |
+
},
|
| 612 |
+
{
|
| 613 |
+
"params": [p for n, p in model.named_parameters() if
|
| 614 |
+
p.requires_grad and any(nd in n for nd in no_decay)],
|
| 615 |
+
"weight_decay": 0.0,
|
| 616 |
+
},
|
| 617 |
+
]
|
| 618 |
+
|
| 619 |
+
optimizer_type = config.optimizer.name
|
| 620 |
+
if optimizer_type == "adamw":
|
| 621 |
+
optimizer = AdamW(
|
| 622 |
+
optimizer_grouped_parameters,
|
| 623 |
+
lr=optimizer_config.learning_rate,
|
| 624 |
+
betas=(optimizer_config.beta1, optimizer_config.beta2),
|
| 625 |
+
weight_decay=optimizer_config.weight_decay,
|
| 626 |
+
eps=optimizer_config.epsilon,
|
| 627 |
+
)
|
| 628 |
+
else:
|
| 629 |
+
raise ValueError(f"Optimizer {optimizer_type} not supported")
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
logger.info("Preprocessing Data")
|
| 633 |
+
|
| 634 |
+
with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f:
|
| 635 |
+
dataset_load = json.load(f)
|
| 636 |
+
|
| 637 |
+
dataset_lm = RLVariantDatasetLLADAV(dataset_load, config)
|
| 638 |
+
|
| 639 |
+
logger.info("Preparing model, optimizer and dataloaders")
|
| 640 |
+
|
| 641 |
+
total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
|
| 642 |
+
num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm)
|
| 643 |
+
num_train_epochs = config.training.num_train_epochs
|
| 644 |
+
max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
|
| 645 |
+
|
| 646 |
+
lr_scheduler = get_scheduler(
|
| 647 |
+
config.lr_scheduler.scheduler,
|
| 648 |
+
optimizer=optimizer,
|
| 649 |
+
num_training_steps=max_train_steps,
|
| 650 |
+
num_warmup_steps=config.lr_scheduler.params.warmup_steps,
|
| 651 |
+
min_lr_scale=config.lr_scheduler.params.min_lr_scale
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
train_dataloader_lm = DataLoader(
|
| 655 |
+
dataset_lm,
|
| 656 |
+
batch_size=config.training.batch_size_lm,
|
| 657 |
+
sampler=None,
|
| 658 |
+
collate_fn=collate_variants,
|
| 659 |
+
num_workers=0,
|
| 660 |
+
shuffle=False
|
| 661 |
+
)
|
| 662 |
+
|
| 663 |
+
model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare(
|
| 664 |
+
model, optimizer, lr_scheduler, train_dataloader_lm
|
| 665 |
+
)
|
| 666 |
+
|
| 667 |
+
logger.info("***** Running inference *****")
|
| 668 |
+
compute_logp_old_tok_parallel(
|
| 669 |
+
accelerator,
|
| 670 |
+
model,
|
| 671 |
+
tokenizer,
|
| 672 |
+
image_processor,
|
| 673 |
+
dataset_lm,
|
| 674 |
+
train_dataloader_lm,
|
| 675 |
+
mask_id=mask_id,
|
| 676 |
+
config=config
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
logger.info("***** Running training *****")
|
| 680 |
+
logger.info(f" Num variants = {len(dataset_lm)}")
|
| 681 |
+
logger.info(f" Num training steps = {max_train_steps}")
|
| 682 |
+
logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
|
| 683 |
+
logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
|
| 684 |
+
logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
|
| 685 |
+
|
| 686 |
+
first_epoch = 0
|
| 687 |
+
data_time_m = AverageMeter()
|
| 688 |
+
end = time.time()
|
| 689 |
+
grad_accum = config.training.gradient_accumulation_steps
|
| 690 |
+
|
| 691 |
+
def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok):
|
| 692 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
|
| 693 |
+
logits = model.lm_head(outputs.last_hidden_state).float()
|
| 694 |
+
|
| 695 |
+
B, T, V = logits.shape
|
| 696 |
+
adv = torch.as_tensor(adv, device=noisy_embeds.device).detach()
|
| 697 |
+
|
| 698 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 699 |
+
safe_labels = labels.clone()
|
| 700 |
+
safe_labels[labels == -100] = 0
|
| 701 |
+
logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
|
| 702 |
+
|
| 703 |
+
ratio = torch.exp(logp_new_tok - logp_old_tok)
|
| 704 |
+
clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps)
|
| 705 |
+
adv_tok = adv.unsqueeze(1)
|
| 706 |
+
|
| 707 |
+
surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok)
|
| 708 |
+
surrogate_tok = surrogate_tok * p_mask_lm
|
| 709 |
+
num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1)
|
| 710 |
+
surrogate_tok = surrogate_tok.sum(dim=1) / num_mask
|
| 711 |
+
policy_loss = - (surrogate_tok.sum() / B)
|
| 712 |
+
|
| 713 |
+
# KL penalty
|
| 714 |
+
kl_loss = torch.tensor(0.0, device=policy_loss.device)
|
| 715 |
+
if config.training.beta > 0:
|
| 716 |
+
kl_seq = logp_new_tok - logp_old_tok
|
| 717 |
+
if config.training.use_kl_estimator_k3:
|
| 718 |
+
kl_seq = (-kl_seq).exp() - 1.0 + kl_seq
|
| 719 |
+
kl_seq = (kl_seq * p_mask_lm).sum(dim=1)
|
| 720 |
+
kl_loss = config.training.beta * kl_seq.sum() / B
|
| 721 |
+
total_loss = policy_loss + kl_loss
|
| 722 |
+
else:
|
| 723 |
+
total_loss = policy_loss
|
| 724 |
+
|
| 725 |
+
return total_loss
|
| 726 |
+
|
| 727 |
+
from tqdm.auto import tqdm
|
| 728 |
+
|
| 729 |
+
cprint("start rl training...", "red")
|
| 730 |
+
for epoch in range(first_epoch, num_train_epochs):
|
| 731 |
+
model.train()
|
| 732 |
+
progress_bar = tqdm(
|
| 733 |
+
train_dataloader_lm,
|
| 734 |
+
desc=f"Epoch {epoch+1}/{num_train_epochs}",
|
| 735 |
+
disable=not accelerator.is_local_main_process,
|
| 736 |
+
dynamic_ncols=True,
|
| 737 |
+
leave=True
|
| 738 |
+
)
|
| 739 |
+
|
| 740 |
+
step = 0
|
| 741 |
+
for step, batch in enumerate(progress_bar, start=1):
|
| 742 |
+
data_time_m.update(time.time() - end)
|
| 743 |
+
|
| 744 |
+
out = prepare_batch_noisy_embeds_rl(
|
| 745 |
+
accelerator=accelerator,
|
| 746 |
+
model=model,
|
| 747 |
+
tokenizer=tokenizer,
|
| 748 |
+
image_processor=image_processor,
|
| 749 |
+
batch=batch,
|
| 750 |
+
mask_id=mask_id,
|
| 751 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 752 |
+
)
|
| 753 |
+
|
| 754 |
+
if out is None:
|
| 755 |
+
continue
|
| 756 |
+
|
| 757 |
+
noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out
|
| 758 |
+
|
| 759 |
+
# Align old log probabilities shape with current noisy_embeds
|
| 760 |
+
L_current = noisy_embeds.shape[1]
|
| 761 |
+
old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device)
|
| 762 |
+
for i, idx in enumerate(valid_idxs):
|
| 763 |
+
old_t = dataset_lm.logp_old_tok[idx]
|
| 764 |
+
l_min = min(L_current, old_t.shape[0])
|
| 765 |
+
old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device)
|
| 766 |
+
|
| 767 |
+
loss_lm = forward_process(
|
| 768 |
+
noisy_embeds=noisy_embeds,
|
| 769 |
+
labels=labels,
|
| 770 |
+
p_mask_lm=p_mask_lm,
|
| 771 |
+
attn_mask=attn_mask,
|
| 772 |
+
adv=rewards,
|
| 773 |
+
logp_old_tok=old_lp_batch
|
| 774 |
+
)
|
| 775 |
+
|
| 776 |
+
loss_lm = loss_lm / grad_accum
|
| 777 |
+
accelerator.backward(loss_lm)
|
| 778 |
+
|
| 779 |
+
if step % grad_accum == 0:
|
| 780 |
+
if config.training.max_grad_norm is not None:
|
| 781 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 782 |
+
optimizer.step()
|
| 783 |
+
lr_scheduler.step()
|
| 784 |
+
optimizer.zero_grad(set_to_none=True)
|
| 785 |
+
torch.cuda.empty_cache()
|
| 786 |
+
|
| 787 |
+
# Catch remaining gradients
|
| 788 |
+
if step % grad_accum != 0:
|
| 789 |
+
if config.training.max_grad_norm is not None:
|
| 790 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 791 |
+
optimizer.step()
|
| 792 |
+
lr_scheduler.step()
|
| 793 |
+
optimizer.zero_grad(set_to_none=True)
|
| 794 |
+
torch.cuda.empty_cache()
|
| 795 |
+
|
| 796 |
+
accelerator.wait_for_everyone()
|
| 797 |
+
|
| 798 |
+
save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
|
| 799 |
+
if config.experiment.current_epoch % config.experiment.save_every == 0:
|
| 800 |
+
save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}")
|
| 801 |
+
|
| 802 |
+
accelerator.end_training()
|
| 803 |
+
|
| 804 |
+
|
| 805 |
+
if __name__ == "__main__":
|
| 806 |
+
main()
|
wandb/run-20260323_031014-fmwqi7oc/files/config.yaml
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.0
|
| 4 |
+
code_path: code/train/rl_lladav.py
|
| 5 |
+
e:
|
| 6 |
+
55pvqnbadfj4j7x2fc9exogdpb6eojxo:
|
| 7 |
+
args:
|
| 8 |
+
- config=configs/rl_lladav_video_reversal.yaml
|
| 9 |
+
- experiment.current_epoch=1
|
| 10 |
+
codePath: train/rl_lladav.py
|
| 11 |
+
codePathLocal: train/rl_lladav.py
|
| 12 |
+
cpu_count: 96
|
| 13 |
+
cpu_count_logical: 192
|
| 14 |
+
cudaVersion: "12.4"
|
| 15 |
+
disk:
|
| 16 |
+
/:
|
| 17 |
+
total: "806578724864"
|
| 18 |
+
used: "571723153408"
|
| 19 |
+
email: shixi_runqing@staff.weibo.com
|
| 20 |
+
executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10
|
| 21 |
+
git:
|
| 22 |
+
commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
|
| 23 |
+
remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
|
| 24 |
+
gpu: NVIDIA H800
|
| 25 |
+
gpu_count: 8
|
| 26 |
+
gpu_nvidia:
|
| 27 |
+
- architecture: Hopper
|
| 28 |
+
cudaCores: 16896
|
| 29 |
+
memoryTotal: "85520809984"
|
| 30 |
+
name: NVIDIA H800
|
| 31 |
+
uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
|
| 32 |
+
- architecture: Hopper
|
| 33 |
+
cudaCores: 16896
|
| 34 |
+
memoryTotal: "85520809984"
|
| 35 |
+
name: NVIDIA H800
|
| 36 |
+
uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
|
| 37 |
+
- architecture: Hopper
|
| 38 |
+
cudaCores: 16896
|
| 39 |
+
memoryTotal: "85520809984"
|
| 40 |
+
name: NVIDIA H800
|
| 41 |
+
uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
|
| 42 |
+
- architecture: Hopper
|
| 43 |
+
cudaCores: 16896
|
| 44 |
+
memoryTotal: "85520809984"
|
| 45 |
+
name: NVIDIA H800
|
| 46 |
+
uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
|
| 47 |
+
- architecture: Hopper
|
| 48 |
+
cudaCores: 16896
|
| 49 |
+
memoryTotal: "85520809984"
|
| 50 |
+
name: NVIDIA H800
|
| 51 |
+
uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
|
| 52 |
+
- architecture: Hopper
|
| 53 |
+
cudaCores: 16896
|
| 54 |
+
memoryTotal: "85520809984"
|
| 55 |
+
name: NVIDIA H800
|
| 56 |
+
uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
|
| 57 |
+
- architecture: Hopper
|
| 58 |
+
cudaCores: 16896
|
| 59 |
+
memoryTotal: "85520809984"
|
| 60 |
+
name: NVIDIA H800
|
| 61 |
+
uuid: GPU-2775489f-18c8-2328-8829-c46238575379
|
| 62 |
+
- architecture: Hopper
|
| 63 |
+
cudaCores: 16896
|
| 64 |
+
memoryTotal: "85520809984"
|
| 65 |
+
name: NVIDIA H800
|
| 66 |
+
uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
|
| 67 |
+
host: b1301b38c75f
|
| 68 |
+
memory:
|
| 69 |
+
total: "2164113813504"
|
| 70 |
+
os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
|
| 71 |
+
program: /data4/runqing/dLLM-RL/train/rl_lladav.py
|
| 72 |
+
python: CPython 3.10.19
|
| 73 |
+
root: /data4/runqing/dLLM-RL
|
| 74 |
+
startedAt: "2026-03-22T19:10:14.663079Z"
|
| 75 |
+
writerId: 55pvqnbadfj4j7x2fc9exogdpb6eojxo
|
| 76 |
+
m: []
|
| 77 |
+
python_version: 3.10.19
|
| 78 |
+
t:
|
| 79 |
+
"1":
|
| 80 |
+
- 1
|
| 81 |
+
- 5
|
| 82 |
+
- 11
|
| 83 |
+
- 41
|
| 84 |
+
- 49
|
| 85 |
+
- 53
|
| 86 |
+
- 71
|
| 87 |
+
- 98
|
| 88 |
+
"2":
|
| 89 |
+
- 1
|
| 90 |
+
- 5
|
| 91 |
+
- 11
|
| 92 |
+
- 41
|
| 93 |
+
- 49
|
| 94 |
+
- 51
|
| 95 |
+
- 53
|
| 96 |
+
- 71
|
| 97 |
+
- 98
|
| 98 |
+
"3":
|
| 99 |
+
- 13
|
| 100 |
+
- 14
|
| 101 |
+
"4": 3.10.19
|
| 102 |
+
"5": 0.25.0
|
| 103 |
+
"6": 4.52.4
|
| 104 |
+
"12": 0.25.0
|
| 105 |
+
"13": linux-x86_64
|
| 106 |
+
config:
|
| 107 |
+
value: configs/rl_lladav_video_reversal.yaml
|
| 108 |
+
dataset.data_type:
|
| 109 |
+
value: video_reverse
|
| 110 |
+
dataset.optimization_data:
|
| 111 |
+
value: rl_data
|
| 112 |
+
dataset.train_dataset:
|
| 113 |
+
value: video_reversal_rl_train
|
| 114 |
+
evaluation.batch_size:
|
| 115 |
+
value: 1
|
| 116 |
+
evaluation.block_size:
|
| 117 |
+
value: 32
|
| 118 |
+
evaluation.data_type:
|
| 119 |
+
value: video_reverse
|
| 120 |
+
evaluation.dynamic_threshold:
|
| 121 |
+
value: 0.9
|
| 122 |
+
evaluation.eval_dataset:
|
| 123 |
+
value: video_reversal_rl_train
|
| 124 |
+
evaluation.further_horizon:
|
| 125 |
+
value: 128
|
| 126 |
+
evaluation.if_eval:
|
| 127 |
+
value: true
|
| 128 |
+
evaluation.max_gen_length:
|
| 129 |
+
value: 256
|
| 130 |
+
evaluation.num_response_per_task:
|
| 131 |
+
value: 3
|
| 132 |
+
evaluation.remasking_strategy.0:
|
| 133 |
+
value: low_confidence_static
|
| 134 |
+
evaluation.remasking_strategy.1:
|
| 135 |
+
value: low_confidence_dynamic
|
| 136 |
+
evaluation.steps:
|
| 137 |
+
value: 256
|
| 138 |
+
evaluation.target:
|
| 139 |
+
value: confidence
|
| 140 |
+
evaluation.temperature:
|
| 141 |
+
value: 0.1
|
| 142 |
+
evaluation.use_cache:
|
| 143 |
+
value: true
|
| 144 |
+
execute.num_chunk:
|
| 145 |
+
value: 128
|
| 146 |
+
experiment.current_epoch:
|
| 147 |
+
value: 1
|
| 148 |
+
experiment.deepspeed_file:
|
| 149 |
+
value: 1_node_8_gpus_deepspeed_zero2_rl
|
| 150 |
+
experiment.eval_every:
|
| 151 |
+
value: 5
|
| 152 |
+
experiment.function:
|
| 153 |
+
value: train
|
| 154 |
+
experiment.logging_dir:
|
| 155 |
+
value: rl_lladav_video_reversal/logs
|
| 156 |
+
experiment.node_index:
|
| 157 |
+
value: 0
|
| 158 |
+
experiment.num_node:
|
| 159 |
+
value: 1
|
| 160 |
+
experiment.project:
|
| 161 |
+
value: rl_lladav_video_reversal
|
| 162 |
+
experiment.save_every:
|
| 163 |
+
value: 10
|
| 164 |
+
experiment.start_from_scratch:
|
| 165 |
+
value: true
|
| 166 |
+
experiment.total_step:
|
| 167 |
+
value: 160
|
| 168 |
+
lr_scheduler.params.learning_rate:
|
| 169 |
+
value: 1e-06
|
| 170 |
+
lr_scheduler.params.min_lr_scale:
|
| 171 |
+
value: 1
|
| 172 |
+
lr_scheduler.params.warmup_steps:
|
| 173 |
+
value: 0
|
| 174 |
+
lr_scheduler.scheduler:
|
| 175 |
+
value: cosine
|
| 176 |
+
model.image_resolution:
|
| 177 |
+
value: 512
|
| 178 |
+
model.model_base:
|
| 179 |
+
value: lladav
|
| 180 |
+
model.optimized_name:
|
| 181 |
+
value: optimized
|
| 182 |
+
model.pretrained_model:
|
| 183 |
+
value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
|
| 184 |
+
model.vq_model_path:
|
| 185 |
+
value: /data4/runqing/hf/magvitv2
|
| 186 |
+
optimizer.name:
|
| 187 |
+
value: adamw
|
| 188 |
+
optimizer.params.beta1:
|
| 189 |
+
value: 0.9
|
| 190 |
+
optimizer.params.beta2:
|
| 191 |
+
value: 0.999
|
| 192 |
+
optimizer.params.epsilon:
|
| 193 |
+
value: 1e-08
|
| 194 |
+
optimizer.params.learning_rate:
|
| 195 |
+
value: 1e-06
|
| 196 |
+
optimizer.params.scale_lr:
|
| 197 |
+
value: false
|
| 198 |
+
optimizer.params.weight_decay:
|
| 199 |
+
value: 0
|
| 200 |
+
reward.answer_must_in_box:
|
| 201 |
+
value: true
|
| 202 |
+
reward.strict_len_check:
|
| 203 |
+
value: false
|
| 204 |
+
rollout.batch_size:
|
| 205 |
+
value: 1
|
| 206 |
+
rollout.block_size:
|
| 207 |
+
value: 32
|
| 208 |
+
rollout.dynamic_threshold:
|
| 209 |
+
value: 0.9
|
| 210 |
+
rollout.further_horizon:
|
| 211 |
+
value: 128
|
| 212 |
+
rollout.max_gen_length:
|
| 213 |
+
value: 128
|
| 214 |
+
rollout.num_response_per_task:
|
| 215 |
+
value: 8
|
| 216 |
+
rollout.num_task_per_step:
|
| 217 |
+
value: 128
|
| 218 |
+
rollout.remasking_strategy:
|
| 219 |
+
value: low_confidence_dynamic
|
| 220 |
+
rollout.steps:
|
| 221 |
+
value: 128
|
| 222 |
+
rollout.target:
|
| 223 |
+
value: confidence
|
| 224 |
+
rollout.temperature:
|
| 225 |
+
value: 0.8
|
| 226 |
+
rollout.use_cache:
|
| 227 |
+
value: true
|
| 228 |
+
system.HF_HOME:
|
| 229 |
+
value: null
|
| 230 |
+
system.HTTP_PROXY:
|
| 231 |
+
value: null
|
| 232 |
+
system.base_dir:
|
| 233 |
+
value: /data4/runqing/dLLM-RL
|
| 234 |
+
system.env_name:
|
| 235 |
+
value: /data8/runqing/miniconda/envs/dllm-rl
|
| 236 |
+
system.envs_dir:
|
| 237 |
+
value: /data8/runqing/miniconda/envs/dllm-rl
|
| 238 |
+
training.batch_size_lm:
|
| 239 |
+
value: 2
|
| 240 |
+
training.beta:
|
| 241 |
+
value: 0.01
|
| 242 |
+
training.enable_tf32:
|
| 243 |
+
value: true
|
| 244 |
+
training.eps:
|
| 245 |
+
value: 0.2
|
| 246 |
+
training.gradient_accumulation_steps:
|
| 247 |
+
value: 64
|
| 248 |
+
training.lower_p:
|
| 249 |
+
value: 0.35
|
| 250 |
+
training.mask_times_per_sample:
|
| 251 |
+
value: 2
|
| 252 |
+
training.max_gen_length:
|
| 253 |
+
value: 128
|
| 254 |
+
training.max_grad_norm:
|
| 255 |
+
value: 1
|
| 256 |
+
training.max_prompt_len:
|
| 257 |
+
value: 512
|
| 258 |
+
training.method:
|
| 259 |
+
value: random_masking
|
| 260 |
+
training.mixed_precision:
|
| 261 |
+
value: bf16
|
| 262 |
+
training.num_train_epochs:
|
| 263 |
+
value: 1
|
| 264 |
+
training.post_num:
|
| 265 |
+
value: 1
|
| 266 |
+
training.seed:
|
| 267 |
+
value: 10086
|
| 268 |
+
training.shrink:
|
| 269 |
+
value: 8
|
| 270 |
+
training.upper_p:
|
| 271 |
+
value: 0.9
|
| 272 |
+
training.use_kl_estimator_k3:
|
| 273 |
+
value: true
|
| 274 |
+
wandb.entity:
|
| 275 |
+
value: null
|
| 276 |
+
wandb.resume:
|
| 277 |
+
value: auto
|
| 278 |
+
wandb.run_id:
|
| 279 |
+
value: fmwqi7oc
|
wandb/run-20260323_031014-fmwqi7oc/files/diff.patch
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..5701c74 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,62 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [],
|
| 158 |
+
+ "source": [
|
| 159 |
+
+ "import json\n",
|
| 160 |
+
+ "\n",
|
| 161 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
|
| 162 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 163 |
+
+ "save_data_list = []\n",
|
| 164 |
+
+ "save_num = 0\n",
|
| 165 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 166 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 167 |
+
+ " raw_data = json.load(fin)\n",
|
| 168 |
+
+ " for data in raw_data:\n",
|
| 169 |
+
+ " if save_num < 100:\n",
|
| 170 |
+
+ " save_data_list.append(data)\n",
|
| 171 |
+
+ " save_num += 1\n",
|
| 172 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 173 |
+
+ " print(len(save_data_list))\n"
|
| 174 |
+
+ ]
|
| 175 |
+
}
|
| 176 |
+
],
|
| 177 |
+
"metadata": {
|
| 178 |
+
@@ -85,7 +141,7 @@
|
| 179 |
+
"name": "python",
|
| 180 |
+
"nbconvert_exporter": "python",
|
| 181 |
+
"pygments_lexer": "ipython3",
|
| 182 |
+
- "version": "3.10.18"
|
| 183 |
+
+ "version": "3.10.19"
|
| 184 |
+
}
|
| 185 |
+
},
|
| 186 |
+
"nbformat": 4,
|
| 187 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 188 |
+
index 8f2c0f1..79d2e5b 100644
|
| 189 |
+
--- a/rl_v.py
|
| 190 |
+
+++ b/rl_v.py
|
| 191 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 192 |
+
def env_prefix(env) -> str:
|
| 193 |
+
return (
|
| 194 |
+
"source ~/.bashrc && "
|
| 195 |
+
- f"source activate {env} && "
|
| 196 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 197 |
+
+ "conda init && "
|
| 198 |
+
+ f"conda activate {env} && "
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
config = get_config()
|
| 202 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 203 |
+
index 4068fc8..d284a23 100644
|
| 204 |
+
--- a/sample/llava/conversation.py
|
| 205 |
+
+++ b/sample/llava/conversation.py
|
| 206 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 207 |
+
if self.version == "llama_v3":
|
| 208 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 209 |
+
elif self.version == "llava_llada":
|
| 210 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 211 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 212 |
+
else:
|
| 213 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 214 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 215 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 216 |
+
index 3db5302..96e33b5 100644
|
| 217 |
+
--- a/sample/llava/model/builder.py
|
| 218 |
+
+++ b/sample/llava/model/builder.py
|
| 219 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 220 |
+
elif "llada" in model_name.lower():
|
| 221 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 222 |
+
|
| 223 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 224 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 225 |
+
+
|
| 226 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 227 |
+
if customized_config is None:
|
| 228 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 229 |
+
else:
|
| 230 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 231 |
+
index 4068fc8..d284a23 100644
|
| 232 |
+
--- a/train/llava/conversation.py
|
| 233 |
+
+++ b/train/llava/conversation.py
|
| 234 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 235 |
+
if self.version == "llama_v3":
|
| 236 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 237 |
+
elif self.version == "llava_llada":
|
| 238 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 239 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 240 |
+
else:
|
| 241 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 242 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 243 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 244 |
+
index 3db5302..aafab95 100644
|
| 245 |
+
--- a/train/llava/model/builder.py
|
| 246 |
+
+++ b/train/llava/model/builder.py
|
| 247 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 248 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 249 |
+
elif "llada" in model_name.lower():
|
| 250 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 251 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 252 |
+
|
| 253 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 254 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 255 |
+
if customized_config is None:
|
| 256 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 257 |
+
else:
|
| 258 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 259 |
+
index 3da28dc..8b17da3 100644
|
| 260 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 261 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 262 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 263 |
+
dpo_forward: Optional[bool] = None,
|
| 264 |
+
cache_position=None,
|
| 265 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 266 |
+
-
|
| 267 |
+
+ conversation_ids = None
|
| 268 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 269 |
+
# donate multi-dialogue
|
| 270 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..5701c74 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,62 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [],
|
| 158 |
+
+ "source": [
|
| 159 |
+
+ "import json\n",
|
| 160 |
+
+ "\n",
|
| 161 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
|
| 162 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 163 |
+
+ "save_data_list = []\n",
|
| 164 |
+
+ "save_num = 0\n",
|
| 165 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 166 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 167 |
+
+ " raw_data = json.load(fin)\n",
|
| 168 |
+
+ " for data in raw_data:\n",
|
| 169 |
+
+ " if save_num < 100:\n",
|
| 170 |
+
+ " save_data_list.append(data)\n",
|
| 171 |
+
+ " save_num += 1\n",
|
| 172 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 173 |
+
+ " print(len(save_data_list))\n"
|
| 174 |
+
+ ]
|
| 175 |
+
}
|
| 176 |
+
],
|
| 177 |
+
"metadata": {
|
| 178 |
+
@@ -85,7 +141,7 @@
|
| 179 |
+
"name": "python",
|
| 180 |
+
"nbconvert_exporter": "python",
|
| 181 |
+
"pygments_lexer": "ipython3",
|
| 182 |
+
- "version": "3.10.18"
|
| 183 |
+
+ "version": "3.10.19"
|
| 184 |
+
}
|
| 185 |
+
},
|
| 186 |
+
"nbformat": 4,
|
| 187 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 188 |
+
index 8f2c0f1..79d2e5b 100644
|
| 189 |
+
--- a/rl_v.py
|
| 190 |
+
+++ b/rl_v.py
|
| 191 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 192 |
+
def env_prefix(env) -> str:
|
| 193 |
+
return (
|
| 194 |
+
"source ~/.bashrc && "
|
| 195 |
+
- f"source activate {env} && "
|
| 196 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 197 |
+
+ "conda init && "
|
| 198 |
+
+ f"conda activate {env} && "
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
config = get_config()
|
| 202 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 203 |
+
index 4068fc8..d284a23 100644
|
| 204 |
+
--- a/sample/llava/conversation.py
|
| 205 |
+
+++ b/sample/llava/conversation.py
|
| 206 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 207 |
+
if self.version == "llama_v3":
|
| 208 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 209 |
+
elif self.version == "llava_llada":
|
| 210 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 211 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 212 |
+
else:
|
| 213 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 214 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 215 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 216 |
+
index 3db5302..96e33b5 100644
|
| 217 |
+
--- a/sample/llava/model/builder.py
|
| 218 |
+
+++ b/sample/llava/model/builder.py
|
| 219 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 220 |
+
elif "llada" in model_name.lower():
|
| 221 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 222 |
+
|
| 223 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 224 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 225 |
+
+
|
| 226 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 227 |
+
if customized_config is None:
|
| 228 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 229 |
+
else:
|
| 230 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 231 |
+
index 4068fc8..d284a23 100644
|
| 232 |
+
--- a/train/llava/conversation.py
|
| 233 |
+
+++ b/train/llava/conversation.py
|
| 234 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 235 |
+
if self.version == "llama_v3":
|
| 236 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 237 |
+
elif self.version == "llava_llada":
|
| 238 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 239 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 240 |
+
else:
|
| 241 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 242 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 243 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 244 |
+
index 3db5302..aafab95 100644
|
| 245 |
+
--- a/train/llava/model/builder.py
|
| 246 |
+
+++ b/train/llava/model/builder.py
|
| 247 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 248 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 249 |
+
elif "llada" in model_name.lower():
|
| 250 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 251 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 252 |
+
|
| 253 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 254 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 255 |
+
if customized_config is None:
|
| 256 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 257 |
+
else:
|
| 258 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 259 |
+
index 3da28dc..8b17da3 100644
|
| 260 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 261 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 262 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 263 |
+
dpo_forward: Optional[bool] = None,
|
| 264 |
+
cache_position=None,
|
| 265 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 266 |
+
-
|
| 267 |
+
+ conversation_ids = None
|
| 268 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 269 |
+
# donate multi-dialogue
|
| 270 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
wandb/run-20260323_031014-fmwqi7oc/files/output.log
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
03/23/2026 03:10:19 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml
|
| 2 |
+
03/23/2026 03:10:19 - INFO - __main__ - Loading LLaDA-V model and tokenizer
|
| 3 |
+
Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
|
| 4 |
+
Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
|
| 5 |
+
Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 5.68s/it]
|
| 6 |
+
Rank 0: Model Class: LlavaLLaDAModelLM
|
| 7 |
+
03/23/2026 03:10:45 - INFO - __main__ - Preprocessing Data
|
| 8 |
+
03/23/2026 03:10:45 - INFO - __main__ - Preparing model, optimizer and dataloaders
|
| 9 |
+
/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
|
| 10 |
+
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
|
| 11 |
+
warnings.warn(
|
| 12 |
+
Before initializing optimizer states
|
| 13 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 14 |
+
CPU Virtual Memory: used = 160.24 GB, percent = 8.0%
|
| 15 |
+
After initializing optimizer states
|
| 16 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 17 |
+
CPU Virtual Memory: used = 186.43 GB, percent = 9.2%
|
| 18 |
+
After initializing ZeRO optimizer
|
| 19 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 20 |
+
CPU Virtual Memory: used = 189.08 GB, percent = 9.4%
|
| 21 |
+
03/23/2026 03:12:00 - INFO - __main__ - ***** Running inference *****
|
| 22 |
+
03/23/2026 03:12:00 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants...
|
| 23 |
+
03/23/2026 03:14:03 - INFO - __main__ - ***** Running training *****
|
| 24 |
+
03/23/2026 03:14:03 - INFO - __main__ - Num variants = 384
|
| 25 |
+
03/23/2026 03:14:03 - INFO - __main__ - Num training steps = 2
|
| 26 |
+
03/23/2026 03:14:03 - INFO - __main__ - Instantaneous batch size per device = 2
|
| 27 |
+
03/23/2026 03:14:03 - INFO - __main__ - Total train batch size (parallel * accum) = 1024
|
| 28 |
+
03/23/2026 03:14:03 - INFO - __main__ - Gradient Accumulation steps = 64
|
| 29 |
+
start rl training...
|
| 30 |
+
Epoch 1/1: 8%|▊ | 2/24 [00:32<06:02, 16.47s/it]
|
| 31 |
+
Traceback (most recent call last):
|
| 32 |
+
File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 806, in <module>
|
| 33 |
+
main()
|
| 34 |
+
File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 767, in main
|
| 35 |
+
loss_lm = forward_process(
|
| 36 |
+
File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 692, in forward_process
|
| 37 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
|
| 38 |
+
File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
|
| 39 |
+
return self._call_impl(*args, **kwargs)
|
| 40 |
+
File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
|
| 41 |
+
return forward_call(*args, **kwargs)
|
| 42 |
+
File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 1027, in forward
|
| 43 |
+
layer_outputs = decoder_layer(
|
| 44 |
+
File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
|
| 45 |
+
return self._call_impl(*args, **kwargs)
|
| 46 |
+
File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
|
| 47 |
+
return forward_call(*args, **kwargs)
|
| 48 |
+
File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 759, in forward
|
| 49 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 50 |
+
File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
|
| 51 |
+
return self._call_impl(*args, **kwargs)
|
| 52 |
+
File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
|
| 53 |
+
return forward_call(*args, **kwargs)
|
| 54 |
+
File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 89, in forward
|
| 55 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 56 |
+
torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 214.00 MiB. GPU 0 has a total capacity of 79.11 GiB of which 208.81 MiB is free. Process 2750357 has 78.90 GiB memory in use. Of the allocated memory 73.41 GiB is allocated by PyTorch, and 1.34 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
| 57 |
+
[rank0]: Traceback (most recent call last):
|
| 58 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 806, in <module>
|
| 59 |
+
[rank0]: main()
|
| 60 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 767, in main
|
| 61 |
+
[rank0]: loss_lm = forward_process(
|
| 62 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 692, in forward_process
|
| 63 |
+
[rank0]: outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
|
| 64 |
+
[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
|
| 65 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 66 |
+
[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
|
| 67 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 68 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 1027, in forward
|
| 69 |
+
[rank0]: layer_outputs = decoder_layer(
|
| 70 |
+
[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
|
| 71 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 72 |
+
[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
|
| 73 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 74 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 759, in forward
|
| 75 |
+
[rank0]: hidden_states = self.post_attention_layernorm(hidden_states)
|
| 76 |
+
[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
|
| 77 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 78 |
+
[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
|
| 79 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 80 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 89, in forward
|
| 81 |
+
[rank0]: variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 82 |
+
[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 214.00 MiB. GPU 0 has a total capacity of 79.11 GiB of which 208.81 MiB is free. Process 2750357 has 78.90 GiB memory in use. Of the allocated memory 73.41 GiB is allocated by PyTorch, and 1.34 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==4.52.4
|
| 2 |
+
nvidia-nvtx-cu12==12.4.127
|
| 3 |
+
deepspeed==0.18.6
|
| 4 |
+
typing-inspection==0.4.2
|
| 5 |
+
scipy==1.15.3
|
| 6 |
+
MarkupSafe==3.0.3
|
| 7 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 8 |
+
yarl==1.22.0
|
| 9 |
+
tornado==6.5.4
|
| 10 |
+
nvidia-curand-cu12==10.3.5.147
|
| 11 |
+
nest-asyncio==1.6.0
|
| 12 |
+
safetensors==0.7.0
|
| 13 |
+
async-timeout==5.0.1
|
| 14 |
+
typing_extensions==4.15.0
|
| 15 |
+
asttokens==3.0.1
|
| 16 |
+
cuda-python==13.1.1
|
| 17 |
+
executing==2.2.1
|
| 18 |
+
nvidia-cudnn-frontend==1.18.0
|
| 19 |
+
comm==0.2.3
|
| 20 |
+
tokenizers==0.21.4
|
| 21 |
+
six==1.17.0
|
| 22 |
+
Pygments==2.19.2
|
| 23 |
+
nvidia-cusparselt-cu12==0.6.2
|
| 24 |
+
apache-tvm-ffi==0.1.8.post2
|
| 25 |
+
flash_attn==2.7.4.post1
|
| 26 |
+
tqdm==4.67.3
|
| 27 |
+
tensorboard-data-server==0.7.2
|
| 28 |
+
annotated-types==0.7.0
|
| 29 |
+
math-verify==0.9.0
|
| 30 |
+
idna==3.11
|
| 31 |
+
regex==2026.1.15
|
| 32 |
+
nvidia-cutlass-dsl-libs-base==4.4.0
|
| 33 |
+
flashinfer-python==0.6.3
|
| 34 |
+
click==8.3.1
|
| 35 |
+
pytz==2025.2
|
| 36 |
+
wheel==0.46.3
|
| 37 |
+
ipykernel==7.2.0
|
| 38 |
+
stack-data==0.6.3
|
| 39 |
+
absl-py==2.4.0
|
| 40 |
+
wandb==0.25.0
|
| 41 |
+
pandas==2.3.3
|
| 42 |
+
termcolor==3.3.0
|
| 43 |
+
sympy==1.13.1
|
| 44 |
+
joblib==1.5.3
|
| 45 |
+
prompt_toolkit==3.0.52
|
| 46 |
+
attrs==25.4.0
|
| 47 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 48 |
+
xxhash==3.6.0
|
| 49 |
+
aiohappyeyeballs==2.6.1
|
| 50 |
+
python-dateutil==2.9.0.post0
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nvidia-ml-py==13.590.48
|
| 53 |
+
multiprocess==0.70.18
|
| 54 |
+
decorator==5.2.1
|
| 55 |
+
cuda-pathfinder==1.3.4
|
| 56 |
+
multidict==6.7.1
|
| 57 |
+
pure_eval==0.2.3
|
| 58 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 59 |
+
omegaconf==2.3.0
|
| 60 |
+
PyYAML==6.0.3
|
| 61 |
+
debugpy==1.8.20
|
| 62 |
+
psutil==7.2.2
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 65 |
+
triton==3.2.0
|
| 66 |
+
antlr4-python3-runtime==4.9.3
|
| 67 |
+
peft==0.18.1
|
| 68 |
+
aiosignal==1.4.0
|
| 69 |
+
accelerate==1.12.0
|
| 70 |
+
liger_kernel==0.7.0
|
| 71 |
+
anyio==4.12.1
|
| 72 |
+
decord==0.6.0
|
| 73 |
+
h11==0.16.0
|
| 74 |
+
urllib3==2.6.3
|
| 75 |
+
wcwidth==0.6.0
|
| 76 |
+
networkx==3.4.2
|
| 77 |
+
huggingface_hub==0.36.2
|
| 78 |
+
tzdata==2025.3
|
| 79 |
+
xformers==0.0.29.post3
|
| 80 |
+
tabulate==0.9.0
|
| 81 |
+
certifi==2026.1.4
|
| 82 |
+
tensorboard==2.20.0
|
| 83 |
+
matplotlib-inline==0.2.1
|
| 84 |
+
pip==26.0.1
|
| 85 |
+
smmap==5.0.2
|
| 86 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 87 |
+
Markdown==3.10.2
|
| 88 |
+
nvidia-nccl-cu12==2.21.5
|
| 89 |
+
aiohttp==3.13.3
|
| 90 |
+
scikit-learn==1.7.2
|
| 91 |
+
nvidia-cutlass-dsl==4.4.0
|
| 92 |
+
setuptools==82.0.0
|
| 93 |
+
jupyter_core==5.9.1
|
| 94 |
+
asyncio==4.0.0
|
| 95 |
+
einops==0.8.2
|
| 96 |
+
pydantic==2.12.5
|
| 97 |
+
av==16.1.0
|
| 98 |
+
protobuf==6.33.5
|
| 99 |
+
evaluate==0.4.6
|
| 100 |
+
Jinja2==3.1.6
|
| 101 |
+
exceptiongroup==1.3.1
|
| 102 |
+
ipython==8.38.0
|
| 103 |
+
sentry-sdk==2.53.0
|
| 104 |
+
httpx==0.28.1
|
| 105 |
+
msgpack==1.1.2
|
| 106 |
+
platformdirs==4.9.2
|
| 107 |
+
hjson==3.1.0
|
| 108 |
+
torchvision==0.21.0
|
| 109 |
+
parso==0.8.6
|
| 110 |
+
cuda-bindings==13.1.1
|
| 111 |
+
pexpect==4.9.0
|
| 112 |
+
threadpoolctl==3.6.0
|
| 113 |
+
pyarrow==23.0.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 116 |
+
Werkzeug==3.1.5
|
| 117 |
+
fsspec==2025.10.0
|
| 118 |
+
pydantic_core==2.41.5
|
| 119 |
+
GitPython==3.1.46
|
| 120 |
+
propcache==0.4.1
|
| 121 |
+
mpmath==1.3.0
|
| 122 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 123 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 124 |
+
hf-xet==1.2.0
|
| 125 |
+
charset-normalizer==3.4.4
|
| 126 |
+
dill==0.4.0
|
| 127 |
+
datasets==4.5.0
|
| 128 |
+
py-cpuinfo==9.0.0
|
| 129 |
+
sentencepiece==0.2.1
|
| 130 |
+
latex2sympy2_extended==1.11.0
|
| 131 |
+
jupyter_client==8.8.0
|
| 132 |
+
pyzmq==27.1.0
|
| 133 |
+
numpy==2.2.6
|
| 134 |
+
gitdb==4.0.12
|
| 135 |
+
grpcio==1.78.0
|
| 136 |
+
packaging==26.0
|
| 137 |
+
requests==2.32.5
|
| 138 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 139 |
+
httpcore==1.0.9
|
| 140 |
+
traitlets==5.14.3
|
| 141 |
+
filelock==3.24.2
|
| 142 |
+
ptyprocess==0.7.0
|
| 143 |
+
pillow==12.1.1
|
| 144 |
+
torch==2.6.0
|
| 145 |
+
more-itertools==10.8.0
|
| 146 |
+
wheel==0.46.3
|
| 147 |
+
jaraco.text==4.0.0
|
| 148 |
+
zipp==3.23.0
|
| 149 |
+
autocommand==2.2.2
|
| 150 |
+
jaraco.context==6.1.0
|
| 151 |
+
jaraco.functools==4.4.0
|
| 152 |
+
tomli==2.4.0
|
| 153 |
+
backports.tarfile==1.2.0
|
| 154 |
+
importlib_metadata==8.7.1
|
| 155 |
+
platformdirs==4.4.0
|
| 156 |
+
packaging==26.0
|
wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-03-22T19:10:14.663079Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"config=configs/rl_lladav_video_reversal.yaml",
|
| 7 |
+
"experiment.current_epoch=1"
|
| 8 |
+
],
|
| 9 |
+
"program": "/data4/runqing/dLLM-RL/train/rl_lladav.py",
|
| 10 |
+
"codePath": "train/rl_lladav.py",
|
| 11 |
+
"codePathLocal": "train/rl_lladav.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
|
| 14 |
+
"commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
|
| 15 |
+
},
|
| 16 |
+
"email": "shixi_runqing@staff.weibo.com",
|
| 17 |
+
"root": "/data4/runqing/dLLM-RL",
|
| 18 |
+
"host": "b1301b38c75f",
|
| 19 |
+
"executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H800",
|
| 23 |
+
"gpu_count": 8,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "806578724864",
|
| 27 |
+
"used": "571723153408"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "2164113813504"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H800",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H800",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H800",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H800",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H800",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H800",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"name": "NVIDIA H800",
|
| 78 |
+
"memoryTotal": "85520809984",
|
| 79 |
+
"cudaCores": 16896,
|
| 80 |
+
"architecture": "Hopper",
|
| 81 |
+
"uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "NVIDIA H800",
|
| 85 |
+
"memoryTotal": "85520809984",
|
| 86 |
+
"cudaCores": 16896,
|
| 87 |
+
"architecture": "Hopper",
|
| 88 |
+
"uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
|
| 89 |
+
}
|
| 90 |
+
],
|
| 91 |
+
"cudaVersion": "12.4",
|
| 92 |
+
"writerId": "55pvqnbadfj4j7x2fc9exogdpb6eojxo"
|
| 93 |
+
}
|
wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_wandb":{"runtime":256},"_runtime":256}
|
wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-23T03:10:14.734689708+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpxva6clnc/port-3523574.txt","pid":3523574,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-03-23T03:10:14.735635443+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3523574}
|
| 3 |
+
{"time":"2026-03-23T03:10:14.735628068+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3523574-3524049-679808348/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-03-23T03:10:14.915970148+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-03-23T03:10:14.923903515+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmwqi7oc","id":"1(@)"}
|
| 6 |
+
{"time":"2026-03-23T03:10:19.250521705+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmwqi7oc","id":"1(@)"}
|
| 7 |
+
{"time":"2026-03-23T03:10:24.490584047+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"srafqf41avyr"}
|
| 8 |
+
{"time":"2026-03-23T03:14:36.354854515+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-03-23T03:14:36.354904386+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-03-23T03:14:36.354956667+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 11 |
+
{"time":"2026-03-23T03:14:36.354918741+08:00","level":"INFO","msg":"server is shutting down"}
|
| 12 |
+
{"time":"2026-03-23T03:14:36.355031381+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3523574-3524049-679808348/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-03-23T03:14:36.574170314+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-03-23T03:14:36.574226901+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-03-23T03:14:36.574272235+08:00","level":"INFO","msg":"server is closed"}
|
wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-23T03:10:14.923991754+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-03-23T03:10:19.249985082+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
|
| 3 |
+
{"time":"2026-03-23T03:10:19.250206083+08:00","level":"INFO","msg":"stream: created new stream","id":"fmwqi7oc"}
|
| 4 |
+
{"time":"2026-03-23T03:10:19.250314544+08:00","level":"INFO","msg":"handler: started","stream_id":"fmwqi7oc"}
|
| 5 |
+
{"time":"2026-03-23T03:10:19.250506873+08:00","level":"INFO","msg":"stream: started","id":"fmwqi7oc"}
|
| 6 |
+
{"time":"2026-03-23T03:10:19.250555978+08:00","level":"INFO","msg":"writer: started","stream_id":"fmwqi7oc"}
|
| 7 |
+
{"time":"2026-03-23T03:10:19.250551745+08:00","level":"INFO","msg":"sender: started","stream_id":"fmwqi7oc"}
|
| 8 |
+
{"time":"2026-03-23T03:10:19.253219433+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
|
| 9 |
+
{"time":"2026-03-23T03:10:24.455218043+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=424ac51772e4e6aeadd1916a4c5be0ccd13e3e923cbb73fc604d2577f3cda919"}
|
| 10 |
+
{"time":"2026-03-23T03:10:24.455395899+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=810d6278f7888329964e292ae4b8d352346389dba5b1e0d16660a92138f8fa0b"}
|
| 11 |
+
{"time":"2026-03-23T03:10:24.455410759+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json</Resource><RequestId>189F3F86588581D5</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=424ac51772e4e6aeadd1916a4c5be0ccd13e3e923cbb73fc604d2577f3cda919, Size: 2635}"}
|
| 12 |
+
{"time":"2026-03-23T03:10:24.455507161+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt</Resource><RequestId>189F3F8658979FBE</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=810d6278f7888329964e292ae4b8d352346389dba5b1e0d16660a92138f8fa0b, Size: 2966}"}
|
| 13 |
+
{"time":"2026-03-23T03:10:24.455771816+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8742bb45572bf89449139797076cdb743a0f0e84b1b2d691043a6f8cab81636"}
|
| 14 |
+
{"time":"2026-03-23T03:10:24.455883557+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189F3F8658A48B88</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8742bb45572bf89449139797076cdb743a0f0e84b1b2d691043a6f8cab81636, Size: 11427}"}
|
| 15 |
+
{"time":"2026-03-23T03:10:24.458949814+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d60641ef66f42e5ff9c0558786e2031e714aa06ed1c87f70bfd6432a5e2bce"}
|
| 16 |
+
{"time":"2026-03-23T03:10:24.459156615+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch</Resource><RequestId>189F3F8658B03704</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d60641ef66f42e5ff9c0558786e2031e714aa06ed1c87f70bfd6432a5e2bce, Size: 11427}"}
|
| 17 |
+
{"time":"2026-03-23T03:10:24.460144111+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b854582f73a02708bfcd8010e9db9ec37b13d12c64988b1b7d8d7a37414bc0e1"}
|
| 18 |
+
{"time":"2026-03-23T03:10:24.460213159+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py</Resource><RequestId>189F3F865877B3DB</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b854582f73a02708bfcd8010e9db9ec37b13d12c64988b1b7d8d7a37414bc0e1, Size: 31804}"}
|
| 19 |
+
{"time":"2026-03-23T03:14:36.354902346+08:00","level":"INFO","msg":"stream: closing","id":"fmwqi7oc"}
|
| 20 |
+
{"time":"2026-03-23T03:14:36.433198106+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=884fe6b1bfe554b7c975118d664e5b487c0bf178608ab506bd9a0e45c348c413"}
|
| 21 |
+
{"time":"2026-03-23T03:14:36.433310818+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log</Resource><RequestId>189F3FC103A8B9D5</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=884fe6b1bfe554b7c975118d664e5b487c0bf178608ab506bd9a0e45c348c413, Size: 7319}"}
|
| 22 |
+
{"time":"2026-03-23T03:14:36.437455015+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba2517cac97729d5e42d029ad94e53f0fdce01e5c8382b91f9f83fad1535086d"}
|
| 23 |
+
{"time":"2026-03-23T03:14:36.437526478+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json</Resource><RequestId>189F3FC103D3486C</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba2517cac97729d5e42d029ad94e53f0fdce01e5c8382b91f9f83fad1535086d, Size: 41}"}
|
| 24 |
+
{"time":"2026-03-23T03:14:36.437757914+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a8c9b732a52ea76e7908bfd3e2f5b5536c95c03abe7dc4d1913e42c397ea578d"}
|
| 25 |
+
{"time":"2026-03-23T03:14:36.437912275+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml</Resource><RequestId>189F3FC103DD3840</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a8c9b732a52ea76e7908bfd3e2f5b5536c95c03abe7dc4d1913e42c397ea578d, Size: 7766}"}
|
| 26 |
+
{"time":"2026-03-23T03:14:36.438391389+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 27 |
+
{"time":"2026-03-23T03:14:36.573284944+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmwqi7oc"}
|
| 28 |
+
{"time":"2026-03-23T03:14:36.573493717+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmwqi7oc"}
|
| 29 |
+
{"time":"2026-03-23T03:14:36.573524005+08:00","level":"INFO","msg":"stream: closed","id":"fmwqi7oc"}
|
wandb/run-20260323_031014-fmwqi7oc/logs/debug.log
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
|
| 2 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Configure stats pid to 3523574
|
| 3 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/logs/debug.log
|
| 5 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log
|
| 6 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}}
|
| 9 |
+
2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-23 03:10:14,916 INFO MainThread:3523574 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-23 03:10:14,920 INFO MainThread:3523574 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-23 03:10:14,923 INFO MainThread:3523574 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-23 03:10:14,928 INFO MainThread:3523574 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-23 03:10:19,377 INFO MainThread:3523574 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-23 03:10:19,489 INFO MainThread:3523574 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-23 03:10:19,490 INFO MainThread:3523574 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmwqi7oc', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 160, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 64, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_train', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 256, 'evaluation.max_gen_length': 256, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'}
|
| 21 |
+
2026-03-23 03:14:36,355 INFO wandb-AsyncioManager-main:3523574 [service_client.py:_forward_responses():134] Reached EOF.
|
| 22 |
+
2026-03-23 03:14:36,355 INFO wandb-AsyncioManager-main:3523574 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb
ADDED
|
Binary file (86.3 kB). View file
|
|
|
wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py
ADDED
|
@@ -0,0 +1,815 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 4 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
| 5 |
+
from datetime import timedelta
|
| 6 |
+
import json
|
| 7 |
+
import logging
|
| 8 |
+
import math
|
| 9 |
+
import shutil
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Dict, List, Any
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from torch.optim import AdamW
|
| 18 |
+
from torch.utils.data import Dataset, DataLoader
|
| 19 |
+
|
| 20 |
+
from omegaconf import OmegaConf
|
| 21 |
+
import wandb
|
| 22 |
+
|
| 23 |
+
from accelerate import Accelerator
|
| 24 |
+
from accelerate.logging import get_logger
|
| 25 |
+
from accelerate.utils import set_seed
|
| 26 |
+
|
| 27 |
+
from train.utils import get_config, flatten_omega_conf, AverageMeter
|
| 28 |
+
from models.lr_schedulers import get_scheduler
|
| 29 |
+
from models.logging import set_verbosity_info, set_verbosity_error
|
| 30 |
+
|
| 31 |
+
from transformers.utils import logging as hf_logging
|
| 32 |
+
hf_logging.set_verbosity_error()
|
| 33 |
+
|
| 34 |
+
from llava.model.builder import load_pretrained_model
|
| 35 |
+
from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord
|
| 36 |
+
from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
|
| 37 |
+
from llava.conversation import conv_templates
|
| 38 |
+
|
| 39 |
+
from termcolor import cprint
|
| 40 |
+
|
| 41 |
+
logger = get_logger(__name__, log_level="INFO")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def collapse_k_unique(lst, k: int):
|
| 45 |
+
if k <= 0:
|
| 46 |
+
raise ValueError("k must be > 0")
|
| 47 |
+
uniq = sorted(set(lst))
|
| 48 |
+
mapping = {}
|
| 49 |
+
n = len(uniq)
|
| 50 |
+
for idx, val in enumerate(uniq):
|
| 51 |
+
group = idx // k
|
| 52 |
+
end_idx = min((group + 1) * k - 1, n - 1)
|
| 53 |
+
rep = uniq[end_idx]
|
| 54 |
+
mapping[val] = rep
|
| 55 |
+
return [mapping[x] for x in lst]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
|
| 59 |
+
import copy
|
| 60 |
+
conv = copy.deepcopy(conv_templates["llava_llada"])
|
| 61 |
+
conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
|
| 62 |
+
conv.append_message(conv.roles[1], response_text)
|
| 63 |
+
return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
|
| 67 |
+
conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
|
| 68 |
+
ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
|
| 69 |
+
return ids.squeeze(0)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
|
| 73 |
+
from PIL import Image
|
| 74 |
+
from types import SimpleNamespace
|
| 75 |
+
|
| 76 |
+
if isinstance(model_cfg, dict):
|
| 77 |
+
model_cfg = SimpleNamespace(**model_cfg)
|
| 78 |
+
|
| 79 |
+
if media_type == "image":
|
| 80 |
+
image = Image.open(abs_path).convert("RGB")
|
| 81 |
+
if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
|
| 82 |
+
setattr(model_cfg, "image_aspect_ratio", "pad")
|
| 83 |
+
if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
|
| 84 |
+
setattr(model_cfg, "mm_patch_merge_type", "flat")
|
| 85 |
+
|
| 86 |
+
image_tensor_list = process_images([image], image_processor, model_cfg)
|
| 87 |
+
image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
|
| 88 |
+
image_sizes = [image.size]
|
| 89 |
+
return image_tensor_list, image_sizes, None
|
| 90 |
+
|
| 91 |
+
elif media_type == "video":
|
| 92 |
+
if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
|
| 93 |
+
if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16)
|
| 94 |
+
if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True)
|
| 95 |
+
|
| 96 |
+
video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
|
| 97 |
+
image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
|
| 98 |
+
image_tensor = image_tensor.to(dtype=dtype, device=device)
|
| 99 |
+
|
| 100 |
+
if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
|
| 101 |
+
v_size = (video_frames.shape[2], video_frames.shape[1])
|
| 102 |
+
else:
|
| 103 |
+
v_size = (0, 0)
|
| 104 |
+
|
| 105 |
+
return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
|
| 109 |
+
out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
|
| 110 |
+
out[-x.numel():] = x
|
| 111 |
+
return out
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
|
| 115 |
+
if x.size(0) == L:
|
| 116 |
+
return x
|
| 117 |
+
out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
|
| 118 |
+
out[-x.size(0):] = x
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
|
| 123 |
+
p = sample.get(f"{media_type}_abs_path")
|
| 124 |
+
if p is None:
|
| 125 |
+
p = sample.get(media_type) or sample.get(f"{media_type}_path")
|
| 126 |
+
if p is None:
|
| 127 |
+
return None
|
| 128 |
+
if not os.path.isabs(p) and dataset_root:
|
| 129 |
+
p = os.path.join(dataset_root, p)
|
| 130 |
+
try:
|
| 131 |
+
return os.path.abspath(p)
|
| 132 |
+
except Exception:
|
| 133 |
+
return p
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class RLVariantDatasetLLADAV(Dataset):
|
| 137 |
+
def __init__(self, samples: List[dict], config):
|
| 138 |
+
super().__init__()
|
| 139 |
+
self.samples = samples
|
| 140 |
+
self.config = config
|
| 141 |
+
self.variants: List[dict] = []
|
| 142 |
+
|
| 143 |
+
# logp_old_tok mapping: idx -> tensor
|
| 144 |
+
self.logp_old_tok: Dict[int, torch.Tensor] = {}
|
| 145 |
+
|
| 146 |
+
self._build_variants()
|
| 147 |
+
|
| 148 |
+
def _build_variants(self):
|
| 149 |
+
sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
|
| 150 |
+
question_prompt = "\nOnly give the best option."
|
| 151 |
+
multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
|
| 152 |
+
|
| 153 |
+
rng = np.random.default_rng(seed=self.config.training.seed or 42)
|
| 154 |
+
idx_counter = 0
|
| 155 |
+
|
| 156 |
+
for sid, s in enumerate(self.samples):
|
| 157 |
+
# Rollout produces "extracted_output" or "response", fall back if needed
|
| 158 |
+
response_text = s.get("response", s.get("extracted_output", ""))
|
| 159 |
+
if isinstance(response_text, list) and len(response_text) > 0:
|
| 160 |
+
response_text = response_text[0] # Just in case it's nested
|
| 161 |
+
|
| 162 |
+
# If options exist (video multiple choice), construct full prompt
|
| 163 |
+
if "options" in s:
|
| 164 |
+
prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt
|
| 165 |
+
else:
|
| 166 |
+
prompt_text = s["prompt"]
|
| 167 |
+
|
| 168 |
+
step_map = s.get("step_map", [])
|
| 169 |
+
|
| 170 |
+
if self.config.training.method == "TraceRL":
|
| 171 |
+
order_list = collapse_k_unique(step_map, self.config.training.shrink)
|
| 172 |
+
uniq_steps = sorted(set(order_list))
|
| 173 |
+
for st in uniq_steps:
|
| 174 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None))
|
| 175 |
+
idx_counter += 1
|
| 176 |
+
|
| 177 |
+
elif self.config.training.method == "random_masking":
|
| 178 |
+
for _ in range(self.config.training.mask_times_per_sample):
|
| 179 |
+
t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
|
| 180 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 181 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed))
|
| 182 |
+
idx_counter += 1
|
| 183 |
+
|
| 184 |
+
elif self.config.training.method == "coupled":
|
| 185 |
+
for _ in range(self.config.training.mask_times_per_sample):
|
| 186 |
+
t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
|
| 187 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 188 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed))
|
| 189 |
+
idx_counter += 1
|
| 190 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed))
|
| 191 |
+
idx_counter += 1
|
| 192 |
+
else:
|
| 193 |
+
raise ValueError(f"Unknown RL method: {self.config.training.method}")
|
| 194 |
+
|
| 195 |
+
def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed):
|
| 196 |
+
return dict(
|
| 197 |
+
idx=idx,
|
| 198 |
+
sample_id=sid,
|
| 199 |
+
prompt=prompt,
|
| 200 |
+
response=response,
|
| 201 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 202 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 203 |
+
reward=s.get("reward", 0.0),
|
| 204 |
+
var_type=var_type,
|
| 205 |
+
var_param=var_param,
|
| 206 |
+
step_map=step_map,
|
| 207 |
+
mask_seed=mask_seed,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
def __len__(self):
|
| 211 |
+
return len(self.variants)
|
| 212 |
+
|
| 213 |
+
def __getitem__(self, idx):
|
| 214 |
+
return self.variants[idx]
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
|
| 218 |
+
out: Dict[str, List[Any]] = {}
|
| 219 |
+
keys = batch[0].keys()
|
| 220 |
+
for k in keys:
|
| 221 |
+
out[k] = [b[k] for b in batch]
|
| 222 |
+
return out
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
@torch.no_grad()
|
| 226 |
+
def prepare_batch_noisy_embeds_rl(
|
| 227 |
+
accelerator,
|
| 228 |
+
model,
|
| 229 |
+
tokenizer,
|
| 230 |
+
image_processor,
|
| 231 |
+
batch: Dict[str, List[Any]],
|
| 232 |
+
mask_id: int,
|
| 233 |
+
dataset_media_root: str | None,
|
| 234 |
+
) -> tuple | None:
|
| 235 |
+
|
| 236 |
+
device = accelerator.device
|
| 237 |
+
bsz = len(batch["idx"])
|
| 238 |
+
|
| 239 |
+
group: Dict[int, List[int]] = {}
|
| 240 |
+
for i, sid in enumerate(batch["sample_id"]):
|
| 241 |
+
group.setdefault(int(sid), []).append(i)
|
| 242 |
+
|
| 243 |
+
sample_cache: Dict[int, Dict[str, Any]] = {}
|
| 244 |
+
for sid, pos_list in group.items():
|
| 245 |
+
first_pos = pos_list[0]
|
| 246 |
+
prompt_text = batch["prompt"][first_pos]
|
| 247 |
+
response_text = batch["response"][first_pos]
|
| 248 |
+
|
| 249 |
+
sample_stub = {
|
| 250 |
+
"image_abs_path": batch.get("image_abs_path", [None])[first_pos],
|
| 251 |
+
"video_abs_path": batch.get("video_abs_path", [None])[first_pos],
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
media_type = None
|
| 255 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
|
| 256 |
+
if p_abs is not None:
|
| 257 |
+
media_type = "video"
|
| 258 |
+
else:
|
| 259 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
|
| 260 |
+
if p_abs is not None:
|
| 261 |
+
media_type = "image"
|
| 262 |
+
|
| 263 |
+
img_list, image_sizes, modalities = None, None, []
|
| 264 |
+
if p_abs is not None:
|
| 265 |
+
img_list, image_sizes, video_stats = load_and_process_media(
|
| 266 |
+
p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
|
| 267 |
+
)
|
| 268 |
+
modalities = [media_type]
|
| 269 |
+
|
| 270 |
+
if media_type == "video" and getattr(model.config, "add_time_instruction", False):
|
| 271 |
+
video_time, frame_time, num_frames = video_stats
|
| 272 |
+
time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
|
| 273 |
+
clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
|
| 274 |
+
prompt_text = f"{time_instruction}\n{clean_prompt}"
|
| 275 |
+
|
| 276 |
+
input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
|
| 277 |
+
end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
|
| 278 |
+
seq_len = len(end_header_seq)
|
| 279 |
+
pos_eh = -1
|
| 280 |
+
if seq_len > 0 and input_ids_text.numel() >= seq_len:
|
| 281 |
+
ids_list = input_ids_text.tolist()
|
| 282 |
+
for i in range(len(ids_list) - seq_len, -1, -1):
|
| 283 |
+
if ids_list[i:i + seq_len] == end_header_seq:
|
| 284 |
+
pos_eh = i + seq_len - 1
|
| 285 |
+
break
|
| 286 |
+
assert pos_eh != -1, " <|end_header_id|> not found"
|
| 287 |
+
start_pos = pos_eh + 1
|
| 288 |
+
|
| 289 |
+
labels_text = input_ids_text.clone()
|
| 290 |
+
labels_text[:start_pos] = -100
|
| 291 |
+
|
| 292 |
+
(_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
|
| 293 |
+
input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
|
| 294 |
+
)
|
| 295 |
+
clean_embeds = clean_embeds.squeeze(0)
|
| 296 |
+
new_labels = new_labels.squeeze(0)
|
| 297 |
+
|
| 298 |
+
eot_id = 126348
|
| 299 |
+
eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
|
| 300 |
+
if eot_indices.numel() > 0:
|
| 301 |
+
last_eot_pos = eot_indices[-1, 0]
|
| 302 |
+
new_labels[last_eot_pos:] = -100
|
| 303 |
+
asst_mask = (new_labels != -100)
|
| 304 |
+
asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1)
|
| 305 |
+
|
| 306 |
+
sample_cache[sid] = dict(
|
| 307 |
+
clean_embeds=clean_embeds,
|
| 308 |
+
new_labels=new_labels,
|
| 309 |
+
asst_idx=asst_idx,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)]
|
| 313 |
+
Lmax = max(lengths)
|
| 314 |
+
mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
|
| 315 |
+
|
| 316 |
+
noisy_list, labels_list, p_mask_list, attn_list = [], [], [], []
|
| 317 |
+
reward_list, valid_idxs = [], []
|
| 318 |
+
|
| 319 |
+
for i in range(bsz):
|
| 320 |
+
sid = int(batch["sample_id"][i])
|
| 321 |
+
var_type = batch["var_type"][i]
|
| 322 |
+
var_param = batch["var_param"][i]
|
| 323 |
+
|
| 324 |
+
clean_embeds = sample_cache[sid]["clean_embeds"].clone()
|
| 325 |
+
new_labels = sample_cache[sid]["new_labels"].clone()
|
| 326 |
+
asst_idx = sample_cache[sid]["asst_idx"]
|
| 327 |
+
|
| 328 |
+
L = clean_embeds.size(0)
|
| 329 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 330 |
+
M = asst_idx.numel()
|
| 331 |
+
|
| 332 |
+
if M == 0:
|
| 333 |
+
continue # skip badly formatted inputs
|
| 334 |
+
|
| 335 |
+
if var_type == "TraceRL":
|
| 336 |
+
step_map = batch["step_map"][i]
|
| 337 |
+
order = torch.as_tensor(step_map, device=device)
|
| 338 |
+
m_len = min(M, order.numel())
|
| 339 |
+
order_full = torch.full((L,), -1, dtype=torch.long, device=device)
|
| 340 |
+
order_full[asst_idx[:m_len]] = order[:m_len]
|
| 341 |
+
step_val = int(var_param)
|
| 342 |
+
|
| 343 |
+
tgt_mask = (order_full == step_val)
|
| 344 |
+
pmask = tgt_mask
|
| 345 |
+
mask_pos = (order_full >= step_val)
|
| 346 |
+
|
| 347 |
+
noisy_embeds = clean_embeds.clone()
|
| 348 |
+
noisy_embeds[mask_pos] = mask_embed
|
| 349 |
+
|
| 350 |
+
elif var_type == "random_masking" or var_type.startswith("coupled"):
|
| 351 |
+
if M == 0:
|
| 352 |
+
# M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward
|
| 353 |
+
noisy_embeds = clean_embeds.clone()
|
| 354 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 355 |
+
else:
|
| 356 |
+
t = float(var_param)
|
| 357 |
+
# 更安全的 seed 获取方式
|
| 358 |
+
mask_seed = batch.get("mask_seed", [None] * bsz)[i]
|
| 359 |
+
if mask_seed is None:
|
| 360 |
+
mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
|
| 361 |
+
|
| 362 |
+
gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
|
| 363 |
+
rnd_cpu = torch.rand(M, generator=gen) < t
|
| 364 |
+
|
| 365 |
+
# 如果是 coupled_comp (对比样本),需要取反
|
| 366 |
+
if var_type == "coupled_comp":
|
| 367 |
+
rnd_cpu = ~rnd_cpu
|
| 368 |
+
|
| 369 |
+
rnd = rnd_cpu.to(device)
|
| 370 |
+
|
| 371 |
+
# 🌟 上帝之手:强行兜底逻辑
|
| 372 |
+
# 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁
|
| 373 |
+
if not rnd.any():
|
| 374 |
+
# 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反,
|
| 375 |
+
# 除非 coupled_rand 全是 True,否则 comp 不可能全 False。
|
| 376 |
+
# 这里统一下一个兜底,非常安全。
|
| 377 |
+
rand_pos = torch.randint(0, M, (1,), device=device)
|
| 378 |
+
rnd[rand_pos] = True
|
| 379 |
+
|
| 380 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 381 |
+
pmask[asst_idx] = rnd
|
| 382 |
+
|
| 383 |
+
noisy_embeds = clean_embeds.clone()
|
| 384 |
+
noisy_embeds[pmask] = mask_embed
|
| 385 |
+
|
| 386 |
+
# elif var_type == "random_masking" or var_type.startswith("coupled"):
|
| 387 |
+
# t = float(var_param)
|
| 388 |
+
# mask_seed = batch["mask_seed"][i]
|
| 389 |
+
# gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
|
| 390 |
+
# rnd_cpu = torch.rand(M, generator=gen) < t
|
| 391 |
+
|
| 392 |
+
# if var_type == "coupled_comp":
|
| 393 |
+
# rnd_cpu = ~rnd_cpu
|
| 394 |
+
|
| 395 |
+
# pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 396 |
+
# pmask[asst_idx] = rnd_cpu.to(device)
|
| 397 |
+
|
| 398 |
+
# noisy_embeds = clean_embeds.clone()
|
| 399 |
+
# noisy_embeds[pmask] = mask_embed
|
| 400 |
+
|
| 401 |
+
if not pmask.any():
|
| 402 |
+
print('************not pmask.any()**********')
|
| 403 |
+
continue
|
| 404 |
+
|
| 405 |
+
noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
|
| 406 |
+
labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
|
| 407 |
+
p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
|
| 408 |
+
attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
|
| 409 |
+
attn[-L:] = True
|
| 410 |
+
attn_list.append(attn)
|
| 411 |
+
reward_list.append(float(batch["reward"][i]))
|
| 412 |
+
valid_idxs.append(batch["idx"][i])
|
| 413 |
+
|
| 414 |
+
if len(noisy_list) == 0:
|
| 415 |
+
return None
|
| 416 |
+
|
| 417 |
+
noisy_batch = torch.stack(noisy_list, dim=0)
|
| 418 |
+
labels_batch = torch.stack(labels_list, dim=0)
|
| 419 |
+
p_mask_batch = torch.stack(p_mask_list, dim=0)
|
| 420 |
+
attn_batch = torch.stack(attn_list, dim=0)
|
| 421 |
+
reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32)
|
| 422 |
+
|
| 423 |
+
return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
@torch.no_grad()
|
| 427 |
+
def compute_logp_old_tok_parallel(
|
| 428 |
+
accelerator,
|
| 429 |
+
model,
|
| 430 |
+
tokenizer,
|
| 431 |
+
image_processor,
|
| 432 |
+
dataset,
|
| 433 |
+
dl,
|
| 434 |
+
mask_id,
|
| 435 |
+
config):
|
| 436 |
+
|
| 437 |
+
model.eval()
|
| 438 |
+
logger.info("Computing reference log probabilities (old_lp) for all variants...")
|
| 439 |
+
|
| 440 |
+
for batch in dl:
|
| 441 |
+
out = prepare_batch_noisy_embeds_rl(
|
| 442 |
+
accelerator=accelerator,
|
| 443 |
+
model=model,
|
| 444 |
+
tokenizer=tokenizer,
|
| 445 |
+
image_processor=image_processor,
|
| 446 |
+
batch=batch,
|
| 447 |
+
mask_id=mask_id,
|
| 448 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 449 |
+
)
|
| 450 |
+
if out is None:
|
| 451 |
+
continue
|
| 452 |
+
|
| 453 |
+
noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out
|
| 454 |
+
|
| 455 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True)
|
| 456 |
+
logits = model.lm_head(outputs.last_hidden_state).float()
|
| 457 |
+
|
| 458 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 459 |
+
safe_labels = labels_batch.clone()
|
| 460 |
+
safe_labels[labels_batch == -100] = 0
|
| 461 |
+
tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
|
| 462 |
+
|
| 463 |
+
for i, idx in enumerate(valid_idxs):
|
| 464 |
+
dataset.logp_old_tok[idx] = tok_lp[i].float().cpu()
|
| 465 |
+
|
| 466 |
+
accelerator.wait_for_everyone()
|
| 467 |
+
model.train()
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
def save_checkpoint(model, tokenizer, config, accelerator, name):
|
| 471 |
+
output_dir = Path(config.experiment.project)
|
| 472 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 473 |
+
|
| 474 |
+
checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
|
| 475 |
+
|
| 476 |
+
if accelerator.is_main_process and checkpoints_total_limit is not None:
|
| 477 |
+
ckpts = sorted(
|
| 478 |
+
[d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
|
| 479 |
+
key=lambda p: int(p.name.split("-")[1]),
|
| 480 |
+
)
|
| 481 |
+
if len(ckpts) >= checkpoints_total_limit:
|
| 482 |
+
to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
|
| 483 |
+
logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
|
| 484 |
+
for p in to_remove:
|
| 485 |
+
shutil.rmtree(p, ignore_errors=True)
|
| 486 |
+
|
| 487 |
+
save_base = output_dir / "ckpt"
|
| 488 |
+
save_base.mkdir(exist_ok=True)
|
| 489 |
+
|
| 490 |
+
model_to_save = accelerator.unwrap_model(model)
|
| 491 |
+
state_dict = accelerator.get_state_dict(model)
|
| 492 |
+
|
| 493 |
+
if accelerator.is_main_process:
|
| 494 |
+
model_to_save.save_pretrained(
|
| 495 |
+
save_base / name,
|
| 496 |
+
save_function=accelerator.save,
|
| 497 |
+
state_dict=state_dict,
|
| 498 |
+
safe_serialization=True,
|
| 499 |
+
)
|
| 500 |
+
tokenizer.save_pretrained(str(save_base / name))
|
| 501 |
+
|
| 502 |
+
metadata = {
|
| 503 |
+
"save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 504 |
+
}
|
| 505 |
+
with (save_base / "metadata.json").open("w") as f:
|
| 506 |
+
json.dump(metadata, f, indent=2)
|
| 507 |
+
|
| 508 |
+
logger.info(f"Saved model + tokenizer to {save_base / name}")
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def main():
|
| 512 |
+
config = get_config()
|
| 513 |
+
|
| 514 |
+
project_name = config.experiment.project
|
| 515 |
+
if config.experiment.current_epoch == 1:
|
| 516 |
+
pretrained_model = config.model.pretrained_model
|
| 517 |
+
else:
|
| 518 |
+
pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name
|
| 519 |
+
|
| 520 |
+
if config.training.enable_tf32:
|
| 521 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 522 |
+
torch.backends.cudnn.benchmark = True
|
| 523 |
+
torch.backends.cudnn.deterministic = False
|
| 524 |
+
|
| 525 |
+
config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
|
| 526 |
+
accelerator = Accelerator(
|
| 527 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 528 |
+
mixed_precision=config.training.mixed_precision,
|
| 529 |
+
log_with='wandb',
|
| 530 |
+
project_dir=config.experiment.logging_dir,
|
| 531 |
+
split_batches=True,
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
logging.basicConfig(
|
| 535 |
+
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
| 536 |
+
datefmt="%m/%d/%Y %H:%M:%S",
|
| 537 |
+
level=logging.INFO,
|
| 538 |
+
)
|
| 539 |
+
logger.info(accelerator.state, main_process_only=False)
|
| 540 |
+
if accelerator.is_local_main_process:
|
| 541 |
+
set_verbosity_info()
|
| 542 |
+
else:
|
| 543 |
+
set_verbosity_error()
|
| 544 |
+
|
| 545 |
+
if accelerator.is_main_process:
|
| 546 |
+
resume_wandb_run = config.wandb.resume
|
| 547 |
+
run_id = config.wandb.get("run_id", None)
|
| 548 |
+
if run_id is None:
|
| 549 |
+
resume_wandb_run = False
|
| 550 |
+
run_id = wandb.util.generate_id()
|
| 551 |
+
config.wandb.run_id = run_id
|
| 552 |
+
|
| 553 |
+
wandb_init_kwargs = dict(
|
| 554 |
+
name=config.experiment.project,
|
| 555 |
+
id=run_id,
|
| 556 |
+
resume=resume_wandb_run,
|
| 557 |
+
entity=config.wandb.get("entity", None),
|
| 558 |
+
config_exclude_keys=[],
|
| 559 |
+
)
|
| 560 |
+
wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
|
| 561 |
+
wandb_config.pop("experiment.resume_from_checkpoint", None)
|
| 562 |
+
|
| 563 |
+
accelerator.init_trackers(
|
| 564 |
+
config.experiment.project,
|
| 565 |
+
config=wandb_config,
|
| 566 |
+
init_kwargs={"wandb": wandb_init_kwargs},
|
| 567 |
+
)
|
| 568 |
+
|
| 569 |
+
if accelerator.is_main_process:
|
| 570 |
+
os.makedirs(config.experiment.project, exist_ok=True)
|
| 571 |
+
config_path = Path(config.experiment.project) / "config.yaml"
|
| 572 |
+
logging.info(f"Saving config to {config_path}")
|
| 573 |
+
OmegaConf.save(config, config_path)
|
| 574 |
+
|
| 575 |
+
if config.training.seed is not None:
|
| 576 |
+
set_seed(config.training.seed)
|
| 577 |
+
|
| 578 |
+
logger.info("Loading LLaDA-V model and tokenizer")
|
| 579 |
+
|
| 580 |
+
tokenizer, model, image_processor, _ = load_pretrained_model(
|
| 581 |
+
pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
|
| 582 |
+
)
|
| 583 |
+
model = model.to(accelerator.device)
|
| 584 |
+
|
| 585 |
+
special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
|
| 586 |
+
num_new = tokenizer.add_special_tokens(special_tokens)
|
| 587 |
+
if num_new > 0:
|
| 588 |
+
model.resize_token_embeddings(len(tokenizer))
|
| 589 |
+
if tokenizer.pad_token is None:
|
| 590 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 591 |
+
|
| 592 |
+
if getattr(config.training, "gradient_checkpointing_enable", False):
|
| 593 |
+
try:
|
| 594 |
+
model.enable_input_require_grads()
|
| 595 |
+
except Exception:
|
| 596 |
+
pass
|
| 597 |
+
try:
|
| 598 |
+
model.gradient_checkpointing_enable()
|
| 599 |
+
except Exception:
|
| 600 |
+
pass
|
| 601 |
+
|
| 602 |
+
mask_id = 126336
|
| 603 |
+
|
| 604 |
+
optimizer_config = config.optimizer.params
|
| 605 |
+
no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
|
| 606 |
+
optimizer_grouped_parameters = [
|
| 607 |
+
{
|
| 608 |
+
"params": [p for n, p in model.named_parameters() if
|
| 609 |
+
p.requires_grad and not any(nd in n for nd in no_decay)],
|
| 610 |
+
"weight_decay": optimizer_config.weight_decay,
|
| 611 |
+
},
|
| 612 |
+
{
|
| 613 |
+
"params": [p for n, p in model.named_parameters() if
|
| 614 |
+
p.requires_grad and any(nd in n for nd in no_decay)],
|
| 615 |
+
"weight_decay": 0.0,
|
| 616 |
+
},
|
| 617 |
+
]
|
| 618 |
+
|
| 619 |
+
optimizer_type = config.optimizer.name
|
| 620 |
+
if optimizer_type == "adamw":
|
| 621 |
+
optimizer = AdamW(
|
| 622 |
+
optimizer_grouped_parameters,
|
| 623 |
+
lr=optimizer_config.learning_rate,
|
| 624 |
+
betas=(optimizer_config.beta1, optimizer_config.beta2),
|
| 625 |
+
weight_decay=optimizer_config.weight_decay,
|
| 626 |
+
eps=optimizer_config.epsilon,
|
| 627 |
+
)
|
| 628 |
+
else:
|
| 629 |
+
raise ValueError(f"Optimizer {optimizer_type} not supported")
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
logger.info("Preprocessing Data")
|
| 633 |
+
|
| 634 |
+
with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f:
|
| 635 |
+
dataset_load = json.load(f)
|
| 636 |
+
|
| 637 |
+
dataset_lm = RLVariantDatasetLLADAV(dataset_load, config)
|
| 638 |
+
|
| 639 |
+
logger.info("Preparing model, optimizer and dataloaders")
|
| 640 |
+
|
| 641 |
+
total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
|
| 642 |
+
num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm)
|
| 643 |
+
num_train_epochs = config.training.num_train_epochs
|
| 644 |
+
max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
|
| 645 |
+
|
| 646 |
+
lr_scheduler = get_scheduler(
|
| 647 |
+
config.lr_scheduler.scheduler,
|
| 648 |
+
optimizer=optimizer,
|
| 649 |
+
num_training_steps=max_train_steps,
|
| 650 |
+
num_warmup_steps=config.lr_scheduler.params.warmup_steps,
|
| 651 |
+
min_lr_scale=config.lr_scheduler.params.min_lr_scale
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
train_dataloader_lm = DataLoader(
|
| 655 |
+
dataset_lm,
|
| 656 |
+
batch_size=config.training.batch_size_lm,
|
| 657 |
+
sampler=None,
|
| 658 |
+
collate_fn=collate_variants,
|
| 659 |
+
num_workers=0,
|
| 660 |
+
shuffle=False
|
| 661 |
+
)
|
| 662 |
+
|
| 663 |
+
model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare(
|
| 664 |
+
model, optimizer, lr_scheduler, train_dataloader_lm
|
| 665 |
+
)
|
| 666 |
+
|
| 667 |
+
logger.info("***** Running inference *****")
|
| 668 |
+
compute_logp_old_tok_parallel(
|
| 669 |
+
accelerator,
|
| 670 |
+
model,
|
| 671 |
+
tokenizer,
|
| 672 |
+
image_processor,
|
| 673 |
+
dataset_lm,
|
| 674 |
+
train_dataloader_lm,
|
| 675 |
+
mask_id=mask_id,
|
| 676 |
+
config=config
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
logger.info("***** Running training *****")
|
| 680 |
+
logger.info(f" Num variants = {len(dataset_lm)}")
|
| 681 |
+
logger.info(f" Num training steps = {max_train_steps}")
|
| 682 |
+
logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
|
| 683 |
+
logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
|
| 684 |
+
logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
|
| 685 |
+
|
| 686 |
+
first_epoch = 0
|
| 687 |
+
data_time_m = AverageMeter()
|
| 688 |
+
end = time.time()
|
| 689 |
+
grad_accum = config.training.gradient_accumulation_steps
|
| 690 |
+
|
| 691 |
+
def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok):
|
| 692 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
|
| 693 |
+
logits = model.lm_head(outputs.last_hidden_state).float()
|
| 694 |
+
|
| 695 |
+
B, T, V = logits.shape
|
| 696 |
+
adv = torch.as_tensor(adv, device=noisy_embeds.device).detach()
|
| 697 |
+
|
| 698 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 699 |
+
safe_labels = labels.clone()
|
| 700 |
+
safe_labels[labels == -100] = 0
|
| 701 |
+
logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
|
| 702 |
+
|
| 703 |
+
ratio = torch.exp(logp_new_tok - logp_old_tok)
|
| 704 |
+
clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps)
|
| 705 |
+
adv_tok = adv.unsqueeze(1)
|
| 706 |
+
|
| 707 |
+
surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok)
|
| 708 |
+
surrogate_tok = surrogate_tok * p_mask_lm
|
| 709 |
+
num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1)
|
| 710 |
+
surrogate_tok = surrogate_tok.sum(dim=1) / num_mask
|
| 711 |
+
policy_loss = - (surrogate_tok.sum() / B)
|
| 712 |
+
|
| 713 |
+
# KL penalty
|
| 714 |
+
kl_loss = torch.tensor(0.0, device=policy_loss.device)
|
| 715 |
+
if config.training.beta > 0:
|
| 716 |
+
kl_seq = logp_new_tok - logp_old_tok
|
| 717 |
+
if config.training.use_kl_estimator_k3:
|
| 718 |
+
kl_seq = (-kl_seq).exp() - 1.0 + kl_seq
|
| 719 |
+
kl_seq = (kl_seq * p_mask_lm).sum(dim=1)
|
| 720 |
+
kl_loss = config.training.beta * kl_seq.sum() / B
|
| 721 |
+
total_loss = policy_loss + kl_loss
|
| 722 |
+
else:
|
| 723 |
+
total_loss = policy_loss
|
| 724 |
+
|
| 725 |
+
return total_loss
|
| 726 |
+
|
| 727 |
+
from tqdm.auto import tqdm
|
| 728 |
+
|
| 729 |
+
cprint("start rl training...", "red")
|
| 730 |
+
for epoch in range(first_epoch, num_train_epochs):
|
| 731 |
+
model.train()
|
| 732 |
+
progress_bar = tqdm(
|
| 733 |
+
train_dataloader_lm,
|
| 734 |
+
desc=f"Epoch {epoch+1}/{num_train_epochs}",
|
| 735 |
+
disable=not accelerator.is_local_main_process,
|
| 736 |
+
dynamic_ncols=True,
|
| 737 |
+
leave=True
|
| 738 |
+
)
|
| 739 |
+
|
| 740 |
+
step = 0
|
| 741 |
+
for step, batch in enumerate(progress_bar, start=1):
|
| 742 |
+
data_time_m.update(time.time() - end)
|
| 743 |
+
|
| 744 |
+
out = prepare_batch_noisy_embeds_rl(
|
| 745 |
+
accelerator=accelerator,
|
| 746 |
+
model=model,
|
| 747 |
+
tokenizer=tokenizer,
|
| 748 |
+
image_processor=image_processor,
|
| 749 |
+
batch=batch,
|
| 750 |
+
mask_id=mask_id,
|
| 751 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 752 |
+
)
|
| 753 |
+
|
| 754 |
+
if out is None:
|
| 755 |
+
continue
|
| 756 |
+
|
| 757 |
+
noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out
|
| 758 |
+
|
| 759 |
+
# Align old log probabilities shape with current noisy_embeds
|
| 760 |
+
L_current = noisy_embeds.shape[1]
|
| 761 |
+
old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device)
|
| 762 |
+
for i, idx in enumerate(valid_idxs):
|
| 763 |
+
old_t = dataset_lm.logp_old_tok[idx]
|
| 764 |
+
l_min = min(L_current, old_t.shape[0])
|
| 765 |
+
old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device)
|
| 766 |
+
|
| 767 |
+
loss_lm = forward_process(
|
| 768 |
+
noisy_embeds=noisy_embeds,
|
| 769 |
+
labels=labels,
|
| 770 |
+
p_mask_lm=p_mask_lm,
|
| 771 |
+
attn_mask=attn_mask,
|
| 772 |
+
adv=rewards,
|
| 773 |
+
logp_old_tok=old_lp_batch
|
| 774 |
+
)
|
| 775 |
+
|
| 776 |
+
loss_lm = loss_lm / grad_accum
|
| 777 |
+
accelerator.backward(loss_lm)
|
| 778 |
+
|
| 779 |
+
if step % grad_accum == 0:
|
| 780 |
+
accelerator.log({
|
| 781 |
+
"train/total_loss": loss_lm.item() * grad_accum,
|
| 782 |
+
"train/policy_loss": p_loss.item(),
|
| 783 |
+
"train/kl_loss": k_loss.item(),
|
| 784 |
+
"train/avg_ratio": avg_ratio.item(),
|
| 785 |
+
"train/learning_rate": current_lr,
|
| 786 |
+
"train/epoch": epoch,
|
| 787 |
+
})
|
| 788 |
+
|
| 789 |
+
if config.training.max_grad_norm is not None:
|
| 790 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 791 |
+
optimizer.step()
|
| 792 |
+
lr_scheduler.step()
|
| 793 |
+
optimizer.zero_grad(set_to_none=True)
|
| 794 |
+
torch.cuda.empty_cache()
|
| 795 |
+
|
| 796 |
+
# Catch remaining gradients
|
| 797 |
+
if step % grad_accum != 0:
|
| 798 |
+
if config.training.max_grad_norm is not None:
|
| 799 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 800 |
+
optimizer.step()
|
| 801 |
+
lr_scheduler.step()
|
| 802 |
+
optimizer.zero_grad(set_to_none=True)
|
| 803 |
+
torch.cuda.empty_cache()
|
| 804 |
+
|
| 805 |
+
accelerator.wait_for_everyone()
|
| 806 |
+
|
| 807 |
+
save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
|
| 808 |
+
if config.experiment.current_epoch % config.experiment.save_every == 0:
|
| 809 |
+
save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}")
|
| 810 |
+
|
| 811 |
+
accelerator.end_training()
|
| 812 |
+
|
| 813 |
+
|
| 814 |
+
if __name__ == "__main__":
|
| 815 |
+
main()
|
wandb/run-20260324_164425-eb44q2tb/files/config.yaml
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.0
|
| 4 |
+
code_path: code/train/rl_lladav.py
|
| 5 |
+
e:
|
| 6 |
+
sr0a7mkap2eiuw8cpbm7o5idgj2l6pki:
|
| 7 |
+
args:
|
| 8 |
+
- config=configs/rl_lladav_video_reversal.yaml
|
| 9 |
+
- experiment.current_epoch=15
|
| 10 |
+
codePath: train/rl_lladav.py
|
| 11 |
+
codePathLocal: train/rl_lladav.py
|
| 12 |
+
cpu_count: 96
|
| 13 |
+
cpu_count_logical: 192
|
| 14 |
+
cudaVersion: "12.4"
|
| 15 |
+
disk:
|
| 16 |
+
/:
|
| 17 |
+
total: "806578724864"
|
| 18 |
+
used: "572337045504"
|
| 19 |
+
email: shixi_runqing@staff.weibo.com
|
| 20 |
+
executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10
|
| 21 |
+
git:
|
| 22 |
+
commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
|
| 23 |
+
remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
|
| 24 |
+
gpu: NVIDIA H800
|
| 25 |
+
gpu_count: 8
|
| 26 |
+
gpu_nvidia:
|
| 27 |
+
- architecture: Hopper
|
| 28 |
+
cudaCores: 16896
|
| 29 |
+
memoryTotal: "85520809984"
|
| 30 |
+
name: NVIDIA H800
|
| 31 |
+
uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
|
| 32 |
+
- architecture: Hopper
|
| 33 |
+
cudaCores: 16896
|
| 34 |
+
memoryTotal: "85520809984"
|
| 35 |
+
name: NVIDIA H800
|
| 36 |
+
uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
|
| 37 |
+
- architecture: Hopper
|
| 38 |
+
cudaCores: 16896
|
| 39 |
+
memoryTotal: "85520809984"
|
| 40 |
+
name: NVIDIA H800
|
| 41 |
+
uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
|
| 42 |
+
- architecture: Hopper
|
| 43 |
+
cudaCores: 16896
|
| 44 |
+
memoryTotal: "85520809984"
|
| 45 |
+
name: NVIDIA H800
|
| 46 |
+
uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
|
| 47 |
+
- architecture: Hopper
|
| 48 |
+
cudaCores: 16896
|
| 49 |
+
memoryTotal: "85520809984"
|
| 50 |
+
name: NVIDIA H800
|
| 51 |
+
uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
|
| 52 |
+
- architecture: Hopper
|
| 53 |
+
cudaCores: 16896
|
| 54 |
+
memoryTotal: "85520809984"
|
| 55 |
+
name: NVIDIA H800
|
| 56 |
+
uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
|
| 57 |
+
- architecture: Hopper
|
| 58 |
+
cudaCores: 16896
|
| 59 |
+
memoryTotal: "85520809984"
|
| 60 |
+
name: NVIDIA H800
|
| 61 |
+
uuid: GPU-2775489f-18c8-2328-8829-c46238575379
|
| 62 |
+
- architecture: Hopper
|
| 63 |
+
cudaCores: 16896
|
| 64 |
+
memoryTotal: "85520809984"
|
| 65 |
+
name: NVIDIA H800
|
| 66 |
+
uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
|
| 67 |
+
host: b1301b38c75f
|
| 68 |
+
memory:
|
| 69 |
+
total: "2164113813504"
|
| 70 |
+
os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
|
| 71 |
+
program: /data4/runqing/dLLM-RL/train/rl_lladav.py
|
| 72 |
+
python: CPython 3.10.19
|
| 73 |
+
root: /data4/runqing/dLLM-RL
|
| 74 |
+
startedAt: "2026-03-24T08:44:25.714950Z"
|
| 75 |
+
writerId: sr0a7mkap2eiuw8cpbm7o5idgj2l6pki
|
| 76 |
+
m: []
|
| 77 |
+
python_version: 3.10.19
|
| 78 |
+
t:
|
| 79 |
+
"1":
|
| 80 |
+
- 1
|
| 81 |
+
- 5
|
| 82 |
+
- 11
|
| 83 |
+
- 41
|
| 84 |
+
- 49
|
| 85 |
+
- 53
|
| 86 |
+
- 71
|
| 87 |
+
- 98
|
| 88 |
+
"2":
|
| 89 |
+
- 1
|
| 90 |
+
- 5
|
| 91 |
+
- 11
|
| 92 |
+
- 41
|
| 93 |
+
- 49
|
| 94 |
+
- 51
|
| 95 |
+
- 53
|
| 96 |
+
- 71
|
| 97 |
+
- 98
|
| 98 |
+
"3":
|
| 99 |
+
- 13
|
| 100 |
+
- 14
|
| 101 |
+
"4": 3.10.19
|
| 102 |
+
"5": 0.25.0
|
| 103 |
+
"6": 4.52.4
|
| 104 |
+
"12": 0.25.0
|
| 105 |
+
"13": linux-x86_64
|
| 106 |
+
config:
|
| 107 |
+
value: configs/rl_lladav_video_reversal.yaml
|
| 108 |
+
dataset.data_type:
|
| 109 |
+
value: video_reverse
|
| 110 |
+
dataset.optimization_data:
|
| 111 |
+
value: rl_data
|
| 112 |
+
dataset.train_dataset:
|
| 113 |
+
value: video_reversal_rl_train
|
| 114 |
+
evaluation.batch_size:
|
| 115 |
+
value: 1
|
| 116 |
+
evaluation.block_size:
|
| 117 |
+
value: 32
|
| 118 |
+
evaluation.data_type:
|
| 119 |
+
value: video_reverse
|
| 120 |
+
evaluation.dynamic_threshold:
|
| 121 |
+
value: 0.9
|
| 122 |
+
evaluation.eval_dataset:
|
| 123 |
+
value: video_reversal_rl_eval
|
| 124 |
+
evaluation.further_horizon:
|
| 125 |
+
value: 128
|
| 126 |
+
evaluation.if_eval:
|
| 127 |
+
value: true
|
| 128 |
+
evaluation.max_gen_length:
|
| 129 |
+
value: 128
|
| 130 |
+
evaluation.num_response_per_task:
|
| 131 |
+
value: 3
|
| 132 |
+
evaluation.remasking_strategy.0:
|
| 133 |
+
value: low_confidence_static
|
| 134 |
+
evaluation.remasking_strategy.1:
|
| 135 |
+
value: low_confidence_dynamic
|
| 136 |
+
evaluation.steps:
|
| 137 |
+
value: 128
|
| 138 |
+
evaluation.target:
|
| 139 |
+
value: confidence
|
| 140 |
+
evaluation.temperature:
|
| 141 |
+
value: 0.1
|
| 142 |
+
evaluation.use_cache:
|
| 143 |
+
value: true
|
| 144 |
+
execute.num_chunk:
|
| 145 |
+
value: 128
|
| 146 |
+
experiment.current_epoch:
|
| 147 |
+
value: 15
|
| 148 |
+
experiment.deepspeed_file:
|
| 149 |
+
value: 1_node_8_gpus_deepspeed_zero2_rl
|
| 150 |
+
experiment.eval_every:
|
| 151 |
+
value: 5
|
| 152 |
+
experiment.function:
|
| 153 |
+
value: train
|
| 154 |
+
experiment.logging_dir:
|
| 155 |
+
value: rl_lladav_video_reversal/logs
|
| 156 |
+
experiment.node_index:
|
| 157 |
+
value: 0
|
| 158 |
+
experiment.num_node:
|
| 159 |
+
value: 1
|
| 160 |
+
experiment.project:
|
| 161 |
+
value: rl_lladav_video_reversal
|
| 162 |
+
experiment.save_every:
|
| 163 |
+
value: 10
|
| 164 |
+
experiment.start_from_scratch:
|
| 165 |
+
value: true
|
| 166 |
+
experiment.total_step:
|
| 167 |
+
value: 20
|
| 168 |
+
lr_scheduler.params.learning_rate:
|
| 169 |
+
value: 1e-06
|
| 170 |
+
lr_scheduler.params.min_lr_scale:
|
| 171 |
+
value: 1
|
| 172 |
+
lr_scheduler.params.warmup_steps:
|
| 173 |
+
value: 0
|
| 174 |
+
lr_scheduler.scheduler:
|
| 175 |
+
value: cosine
|
| 176 |
+
model.image_resolution:
|
| 177 |
+
value: 512
|
| 178 |
+
model.model_base:
|
| 179 |
+
value: lladav
|
| 180 |
+
model.optimized_name:
|
| 181 |
+
value: optimized
|
| 182 |
+
model.pretrained_model:
|
| 183 |
+
value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
|
| 184 |
+
model.vq_model_path:
|
| 185 |
+
value: /data4/runqing/hf/magvitv2
|
| 186 |
+
optimizer.name:
|
| 187 |
+
value: adamw
|
| 188 |
+
optimizer.params.beta1:
|
| 189 |
+
value: 0.9
|
| 190 |
+
optimizer.params.beta2:
|
| 191 |
+
value: 0.999
|
| 192 |
+
optimizer.params.epsilon:
|
| 193 |
+
value: 1e-08
|
| 194 |
+
optimizer.params.learning_rate:
|
| 195 |
+
value: 1e-06
|
| 196 |
+
optimizer.params.scale_lr:
|
| 197 |
+
value: false
|
| 198 |
+
optimizer.params.weight_decay:
|
| 199 |
+
value: 0
|
| 200 |
+
reward.answer_must_in_box:
|
| 201 |
+
value: true
|
| 202 |
+
reward.strict_len_check:
|
| 203 |
+
value: false
|
| 204 |
+
rollout.batch_size:
|
| 205 |
+
value: 1
|
| 206 |
+
rollout.block_size:
|
| 207 |
+
value: 32
|
| 208 |
+
rollout.dynamic_threshold:
|
| 209 |
+
value: 0.9
|
| 210 |
+
rollout.further_horizon:
|
| 211 |
+
value: 128
|
| 212 |
+
rollout.max_gen_length:
|
| 213 |
+
value: 128
|
| 214 |
+
rollout.num_response_per_task:
|
| 215 |
+
value: 8
|
| 216 |
+
rollout.num_task_per_step:
|
| 217 |
+
value: 128
|
| 218 |
+
rollout.remasking_strategy:
|
| 219 |
+
value: low_confidence_dynamic
|
| 220 |
+
rollout.steps:
|
| 221 |
+
value: 128
|
| 222 |
+
rollout.target:
|
| 223 |
+
value: confidence
|
| 224 |
+
rollout.temperature:
|
| 225 |
+
value: 0.8
|
| 226 |
+
rollout.use_cache:
|
| 227 |
+
value: true
|
| 228 |
+
system.HF_HOME:
|
| 229 |
+
value: null
|
| 230 |
+
system.HTTP_PROXY:
|
| 231 |
+
value: null
|
| 232 |
+
system.base_dir:
|
| 233 |
+
value: /data4/runqing/dLLM-RL
|
| 234 |
+
system.env_name:
|
| 235 |
+
value: /data8/runqing/miniconda/envs/dllm-rl
|
| 236 |
+
system.envs_dir:
|
| 237 |
+
value: /data8/runqing/miniconda/envs/dllm-rl
|
| 238 |
+
training.batch_size_lm:
|
| 239 |
+
value: 1
|
| 240 |
+
training.beta:
|
| 241 |
+
value: 0.01
|
| 242 |
+
training.enable_tf32:
|
| 243 |
+
value: true
|
| 244 |
+
training.eps:
|
| 245 |
+
value: 0.2
|
| 246 |
+
training.gradient_accumulation_steps:
|
| 247 |
+
value: 64
|
| 248 |
+
training.lower_p:
|
| 249 |
+
value: 0.35
|
| 250 |
+
training.mask_times_per_sample:
|
| 251 |
+
value: 2
|
| 252 |
+
training.max_gen_length:
|
| 253 |
+
value: 128
|
| 254 |
+
training.max_grad_norm:
|
| 255 |
+
value: 1
|
| 256 |
+
training.max_prompt_len:
|
| 257 |
+
value: 512
|
| 258 |
+
training.method:
|
| 259 |
+
value: random_masking
|
| 260 |
+
training.mixed_precision:
|
| 261 |
+
value: bf16
|
| 262 |
+
training.num_train_epochs:
|
| 263 |
+
value: 1
|
| 264 |
+
training.post_num:
|
| 265 |
+
value: 1
|
| 266 |
+
training.seed:
|
| 267 |
+
value: 10086
|
| 268 |
+
training.shrink:
|
| 269 |
+
value: 8
|
| 270 |
+
training.upper_p:
|
| 271 |
+
value: 0.9
|
| 272 |
+
training.use_kl_estimator_k3:
|
| 273 |
+
value: true
|
| 274 |
+
wandb.entity:
|
| 275 |
+
value: null
|
| 276 |
+
wandb.resume:
|
| 277 |
+
value: auto
|
| 278 |
+
wandb.run_id:
|
| 279 |
+
value: eb44q2tb
|
wandb/run-20260324_164425-eb44q2tb/files/diff.patch
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..9f94fa7 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,87 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [
|
| 158 |
+
+ {
|
| 159 |
+
+ "name": "stdout",
|
| 160 |
+
+ "output_type": "stream",
|
| 161 |
+
+ "text": [
|
| 162 |
+
+ "200\n"
|
| 163 |
+
+ ]
|
| 164 |
+
+ },
|
| 165 |
+
+ {
|
| 166 |
+
+ "ename": "",
|
| 167 |
+
+ "evalue": "",
|
| 168 |
+
+ "output_type": "error",
|
| 169 |
+
+ "traceback": [
|
| 170 |
+
+ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
|
| 171 |
+
+ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
|
| 172 |
+
+ "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
|
| 173 |
+
+ "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
|
| 174 |
+
+ ]
|
| 175 |
+
+ }
|
| 176 |
+
+ ],
|
| 177 |
+
+ "source": [
|
| 178 |
+
+ "import json\n",
|
| 179 |
+
+ "import random\n",
|
| 180 |
+
+ "random.seed(2026)\n",
|
| 181 |
+
+ "\n",
|
| 182 |
+
+ "\n",
|
| 183 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
|
| 184 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
|
| 185 |
+
+ "save_data_list = []\n",
|
| 186 |
+
+ "save_num = 0\n",
|
| 187 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 188 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 189 |
+
+ " raw_data = json.load(fin)\n",
|
| 190 |
+
+ "\n",
|
| 191 |
+
+ " # random.shuffle(raw_data)\n",
|
| 192 |
+
+ " for did, data in enumerate(raw_data):\n",
|
| 193 |
+
+ " # if did > 1999:\n",
|
| 194 |
+
+ " if save_num < 200:\n",
|
| 195 |
+
+ " save_data_list.append(data)\n",
|
| 196 |
+
+ " save_num += 1\n",
|
| 197 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 198 |
+
+ " print(len(save_data_list))\n"
|
| 199 |
+
+ ]
|
| 200 |
+
}
|
| 201 |
+
],
|
| 202 |
+
"metadata": {
|
| 203 |
+
@@ -85,7 +166,7 @@
|
| 204 |
+
"name": "python",
|
| 205 |
+
"nbconvert_exporter": "python",
|
| 206 |
+
"pygments_lexer": "ipython3",
|
| 207 |
+
- "version": "3.10.18"
|
| 208 |
+
+ "version": "3.10.19"
|
| 209 |
+
}
|
| 210 |
+
},
|
| 211 |
+
"nbformat": 4,
|
| 212 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 213 |
+
index 8f2c0f1..79d2e5b 100644
|
| 214 |
+
--- a/rl_v.py
|
| 215 |
+
+++ b/rl_v.py
|
| 216 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 217 |
+
def env_prefix(env) -> str:
|
| 218 |
+
return (
|
| 219 |
+
"source ~/.bashrc && "
|
| 220 |
+
- f"source activate {env} && "
|
| 221 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 222 |
+
+ "conda init && "
|
| 223 |
+
+ f"conda activate {env} && "
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
config = get_config()
|
| 227 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 228 |
+
index 4068fc8..d284a23 100644
|
| 229 |
+
--- a/sample/llava/conversation.py
|
| 230 |
+
+++ b/sample/llava/conversation.py
|
| 231 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 232 |
+
if self.version == "llama_v3":
|
| 233 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 234 |
+
elif self.version == "llava_llada":
|
| 235 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 236 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 237 |
+
else:
|
| 238 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 239 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 240 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 241 |
+
index 3db5302..96e33b5 100644
|
| 242 |
+
--- a/sample/llava/model/builder.py
|
| 243 |
+
+++ b/sample/llava/model/builder.py
|
| 244 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 245 |
+
elif "llada" in model_name.lower():
|
| 246 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 247 |
+
|
| 248 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 249 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 250 |
+
+
|
| 251 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 252 |
+
if customized_config is None:
|
| 253 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 254 |
+
else:
|
| 255 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 256 |
+
index 4068fc8..d284a23 100644
|
| 257 |
+
--- a/train/llava/conversation.py
|
| 258 |
+
+++ b/train/llava/conversation.py
|
| 259 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 260 |
+
if self.version == "llama_v3":
|
| 261 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 262 |
+
elif self.version == "llava_llada":
|
| 263 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 264 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 265 |
+
else:
|
| 266 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 267 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 268 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 269 |
+
index 3db5302..aafab95 100644
|
| 270 |
+
--- a/train/llava/model/builder.py
|
| 271 |
+
+++ b/train/llava/model/builder.py
|
| 272 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 273 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 274 |
+
elif "llada" in model_name.lower():
|
| 275 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 276 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 277 |
+
|
| 278 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 279 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 280 |
+
if customized_config is None:
|
| 281 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 282 |
+
else:
|
| 283 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 284 |
+
index 3da28dc..8b17da3 100644
|
| 285 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 286 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 287 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 288 |
+
dpo_forward: Optional[bool] = None,
|
| 289 |
+
cache_position=None,
|
| 290 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 291 |
+
-
|
| 292 |
+
+ conversation_ids = None
|
| 293 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 294 |
+
# donate multi-dialogue
|
| 295 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
| 296 |
+
diff --git a/train/rl_llada.py b/train/rl_llada.py
|
| 297 |
+
index 916ce23..00693be 100644
|
| 298 |
+
--- a/train/rl_llada.py
|
| 299 |
+
+++ b/train/rl_llada.py
|
| 300 |
+
@@ -96,7 +96,7 @@ def main():
|
| 301 |
+
accelerator = Accelerator(
|
| 302 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 303 |
+
mixed_precision=config.training.mixed_precision,
|
| 304 |
+
- log_with=None,
|
| 305 |
+
+ log_with='wandb',
|
| 306 |
+
project_dir=config.experiment.logging_dir,
|
| 307 |
+
split_batches=True,
|
| 308 |
+
)
|
wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..9f94fa7 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,87 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [
|
| 158 |
+
+ {
|
| 159 |
+
+ "name": "stdout",
|
| 160 |
+
+ "output_type": "stream",
|
| 161 |
+
+ "text": [
|
| 162 |
+
+ "200\n"
|
| 163 |
+
+ ]
|
| 164 |
+
+ },
|
| 165 |
+
+ {
|
| 166 |
+
+ "ename": "",
|
| 167 |
+
+ "evalue": "",
|
| 168 |
+
+ "output_type": "error",
|
| 169 |
+
+ "traceback": [
|
| 170 |
+
+ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
|
| 171 |
+
+ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
|
| 172 |
+
+ "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
|
| 173 |
+
+ "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
|
| 174 |
+
+ ]
|
| 175 |
+
+ }
|
| 176 |
+
+ ],
|
| 177 |
+
+ "source": [
|
| 178 |
+
+ "import json\n",
|
| 179 |
+
+ "import random\n",
|
| 180 |
+
+ "random.seed(2026)\n",
|
| 181 |
+
+ "\n",
|
| 182 |
+
+ "\n",
|
| 183 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
|
| 184 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
|
| 185 |
+
+ "save_data_list = []\n",
|
| 186 |
+
+ "save_num = 0\n",
|
| 187 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 188 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 189 |
+
+ " raw_data = json.load(fin)\n",
|
| 190 |
+
+ "\n",
|
| 191 |
+
+ " # random.shuffle(raw_data)\n",
|
| 192 |
+
+ " for did, data in enumerate(raw_data):\n",
|
| 193 |
+
+ " # if did > 1999:\n",
|
| 194 |
+
+ " if save_num < 200:\n",
|
| 195 |
+
+ " save_data_list.append(data)\n",
|
| 196 |
+
+ " save_num += 1\n",
|
| 197 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 198 |
+
+ " print(len(save_data_list))\n"
|
| 199 |
+
+ ]
|
| 200 |
+
}
|
| 201 |
+
],
|
| 202 |
+
"metadata": {
|
| 203 |
+
@@ -85,7 +166,7 @@
|
| 204 |
+
"name": "python",
|
| 205 |
+
"nbconvert_exporter": "python",
|
| 206 |
+
"pygments_lexer": "ipython3",
|
| 207 |
+
- "version": "3.10.18"
|
| 208 |
+
+ "version": "3.10.19"
|
| 209 |
+
}
|
| 210 |
+
},
|
| 211 |
+
"nbformat": 4,
|
| 212 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 213 |
+
index 8f2c0f1..79d2e5b 100644
|
| 214 |
+
--- a/rl_v.py
|
| 215 |
+
+++ b/rl_v.py
|
| 216 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 217 |
+
def env_prefix(env) -> str:
|
| 218 |
+
return (
|
| 219 |
+
"source ~/.bashrc && "
|
| 220 |
+
- f"source activate {env} && "
|
| 221 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 222 |
+
+ "conda init && "
|
| 223 |
+
+ f"conda activate {env} && "
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
config = get_config()
|
| 227 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 228 |
+
index 4068fc8..d284a23 100644
|
| 229 |
+
--- a/sample/llava/conversation.py
|
| 230 |
+
+++ b/sample/llava/conversation.py
|
| 231 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 232 |
+
if self.version == "llama_v3":
|
| 233 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 234 |
+
elif self.version == "llava_llada":
|
| 235 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 236 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 237 |
+
else:
|
| 238 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 239 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 240 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 241 |
+
index 3db5302..96e33b5 100644
|
| 242 |
+
--- a/sample/llava/model/builder.py
|
| 243 |
+
+++ b/sample/llava/model/builder.py
|
| 244 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 245 |
+
elif "llada" in model_name.lower():
|
| 246 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 247 |
+
|
| 248 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 249 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 250 |
+
+
|
| 251 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 252 |
+
if customized_config is None:
|
| 253 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 254 |
+
else:
|
| 255 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 256 |
+
index 4068fc8..d284a23 100644
|
| 257 |
+
--- a/train/llava/conversation.py
|
| 258 |
+
+++ b/train/llava/conversation.py
|
| 259 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 260 |
+
if self.version == "llama_v3":
|
| 261 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 262 |
+
elif self.version == "llava_llada":
|
| 263 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 264 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 265 |
+
else:
|
| 266 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 267 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 268 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 269 |
+
index 3db5302..aafab95 100644
|
| 270 |
+
--- a/train/llava/model/builder.py
|
| 271 |
+
+++ b/train/llava/model/builder.py
|
| 272 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 273 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 274 |
+
elif "llada" in model_name.lower():
|
| 275 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 276 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 277 |
+
|
| 278 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 279 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 280 |
+
if customized_config is None:
|
| 281 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 282 |
+
else:
|
| 283 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 284 |
+
index 3da28dc..8b17da3 100644
|
| 285 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 286 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 287 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 288 |
+
dpo_forward: Optional[bool] = None,
|
| 289 |
+
cache_position=None,
|
| 290 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 291 |
+
-
|
| 292 |
+
+ conversation_ids = None
|
| 293 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 294 |
+
# donate multi-dialogue
|
| 295 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
| 296 |
+
diff --git a/train/rl_llada.py b/train/rl_llada.py
|
| 297 |
+
index 916ce23..00693be 100644
|
| 298 |
+
--- a/train/rl_llada.py
|
| 299 |
+
+++ b/train/rl_llada.py
|
| 300 |
+
@@ -96,7 +96,7 @@ def main():
|
| 301 |
+
accelerator = Accelerator(
|
| 302 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 303 |
+
mixed_precision=config.training.mixed_precision,
|
| 304 |
+
- log_with=None,
|
| 305 |
+
+ log_with='wandb',
|
| 306 |
+
project_dir=config.experiment.logging_dir,
|
| 307 |
+
split_batches=True,
|
| 308 |
+
)
|
wandb/run-20260324_164425-eb44q2tb/files/output.log
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
03/24/2026 16:44:30 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml
|
| 2 |
+
03/24/2026 16:44:30 - INFO - __main__ - Loading LLaDA-V model and tokenizer
|
| 3 |
+
Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal/ckpt/optimized
|
| 4 |
+
Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
|
| 5 |
+
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.39s/it]
|
| 6 |
+
Rank 0: Model Class: LlavaLLaDAModelLM
|
| 7 |
+
03/24/2026 16:44:52 - INFO - __main__ - Preprocessing Data
|
| 8 |
+
03/24/2026 16:44:52 - INFO - __main__ - Preparing model, optimizer and dataloaders
|
| 9 |
+
/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
|
| 10 |
+
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
|
| 11 |
+
warnings.warn(
|
| 12 |
+
Before initializing optimizer states
|
| 13 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 14 |
+
CPU Virtual Memory: used = 178.31 GB, percent = 8.8%
|
| 15 |
+
After initializing optimizer states
|
| 16 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 17 |
+
CPU Virtual Memory: used = 200.78 GB, percent = 10.0%
|
| 18 |
+
After initializing ZeRO optimizer
|
| 19 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 20 |
+
CPU Virtual Memory: used = 202.31 GB, percent = 10.0%
|
| 21 |
+
03/24/2026 16:46:09 - INFO - __main__ - ***** Running inference *****
|
| 22 |
+
03/24/2026 16:46:09 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants...
|
| 23 |
+
03/24/2026 16:50:40 - INFO - __main__ - ***** Running training *****
|
| 24 |
+
03/24/2026 16:50:40 - INFO - __main__ - Num variants = 704
|
| 25 |
+
03/24/2026 16:50:40 - INFO - __main__ - Num training steps = 3
|
| 26 |
+
03/24/2026 16:50:40 - INFO - __main__ - Instantaneous batch size per device = 1
|
| 27 |
+
03/24/2026 16:50:40 - INFO - __main__ - Total train batch size (parallel * accum) = 512
|
| 28 |
+
03/24/2026 16:50:40 - INFO - __main__ - Gradient Accumulation steps = 64
|
| 29 |
+
start rl training...
|
| 30 |
+
Epoch 1/1: 72%|███████▏ | 63/88 [11:05<04:24, 10.56s/it]
|
| 31 |
+
Traceback (most recent call last):
|
| 32 |
+
File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 815, in <module>
|
| 33 |
+
main()
|
| 34 |
+
File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 782, in main
|
| 35 |
+
"train/policy_loss": p_loss.item(),
|
| 36 |
+
NameError: name 'p_loss' is not defined
|
| 37 |
+
[rank0]: Traceback (most recent call last):
|
| 38 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 815, in <module>
|
| 39 |
+
[rank0]: main()
|
| 40 |
+
[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 782, in main
|
| 41 |
+
[rank0]: "train/policy_loss": p_loss.item(),
|
| 42 |
+
[rank0]: NameError: name 'p_loss' is not defined
|
wandb/run-20260324_164425-eb44q2tb/files/requirements.txt
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==4.52.4
|
| 2 |
+
nvidia-nvtx-cu12==12.4.127
|
| 3 |
+
deepspeed==0.18.6
|
| 4 |
+
typing-inspection==0.4.2
|
| 5 |
+
scipy==1.15.3
|
| 6 |
+
MarkupSafe==3.0.3
|
| 7 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 8 |
+
yarl==1.22.0
|
| 9 |
+
tornado==6.5.4
|
| 10 |
+
nvidia-curand-cu12==10.3.5.147
|
| 11 |
+
nest-asyncio==1.6.0
|
| 12 |
+
safetensors==0.7.0
|
| 13 |
+
async-timeout==5.0.1
|
| 14 |
+
typing_extensions==4.15.0
|
| 15 |
+
asttokens==3.0.1
|
| 16 |
+
cuda-python==13.1.1
|
| 17 |
+
executing==2.2.1
|
| 18 |
+
nvidia-cudnn-frontend==1.18.0
|
| 19 |
+
comm==0.2.3
|
| 20 |
+
tokenizers==0.21.4
|
| 21 |
+
six==1.17.0
|
| 22 |
+
Pygments==2.19.2
|
| 23 |
+
nvidia-cusparselt-cu12==0.6.2
|
| 24 |
+
apache-tvm-ffi==0.1.8.post2
|
| 25 |
+
flash_attn==2.7.4.post1
|
| 26 |
+
tqdm==4.67.3
|
| 27 |
+
tensorboard-data-server==0.7.2
|
| 28 |
+
annotated-types==0.7.0
|
| 29 |
+
math-verify==0.9.0
|
| 30 |
+
idna==3.11
|
| 31 |
+
regex==2026.1.15
|
| 32 |
+
nvidia-cutlass-dsl-libs-base==4.4.0
|
| 33 |
+
flashinfer-python==0.6.3
|
| 34 |
+
click==8.3.1
|
| 35 |
+
pytz==2025.2
|
| 36 |
+
wheel==0.46.3
|
| 37 |
+
ipykernel==7.2.0
|
| 38 |
+
stack-data==0.6.3
|
| 39 |
+
absl-py==2.4.0
|
| 40 |
+
wandb==0.25.0
|
| 41 |
+
pandas==2.3.3
|
| 42 |
+
termcolor==3.3.0
|
| 43 |
+
sympy==1.13.1
|
| 44 |
+
joblib==1.5.3
|
| 45 |
+
prompt_toolkit==3.0.52
|
| 46 |
+
attrs==25.4.0
|
| 47 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 48 |
+
xxhash==3.6.0
|
| 49 |
+
aiohappyeyeballs==2.6.1
|
| 50 |
+
python-dateutil==2.9.0.post0
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nvidia-ml-py==13.590.48
|
| 53 |
+
multiprocess==0.70.18
|
| 54 |
+
decorator==5.2.1
|
| 55 |
+
cuda-pathfinder==1.3.4
|
| 56 |
+
multidict==6.7.1
|
| 57 |
+
pure_eval==0.2.3
|
| 58 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 59 |
+
omegaconf==2.3.0
|
| 60 |
+
PyYAML==6.0.3
|
| 61 |
+
debugpy==1.8.20
|
| 62 |
+
psutil==7.2.2
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 65 |
+
triton==3.2.0
|
| 66 |
+
antlr4-python3-runtime==4.9.3
|
| 67 |
+
peft==0.18.1
|
| 68 |
+
aiosignal==1.4.0
|
| 69 |
+
accelerate==1.12.0
|
| 70 |
+
liger_kernel==0.7.0
|
| 71 |
+
anyio==4.12.1
|
| 72 |
+
decord==0.6.0
|
| 73 |
+
h11==0.16.0
|
| 74 |
+
urllib3==2.6.3
|
| 75 |
+
wcwidth==0.6.0
|
| 76 |
+
networkx==3.4.2
|
| 77 |
+
huggingface_hub==0.36.2
|
| 78 |
+
tzdata==2025.3
|
| 79 |
+
xformers==0.0.29.post3
|
| 80 |
+
tabulate==0.9.0
|
| 81 |
+
certifi==2026.1.4
|
| 82 |
+
tensorboard==2.20.0
|
| 83 |
+
matplotlib-inline==0.2.1
|
| 84 |
+
pip==26.0.1
|
| 85 |
+
smmap==5.0.2
|
| 86 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 87 |
+
Markdown==3.10.2
|
| 88 |
+
nvidia-nccl-cu12==2.21.5
|
| 89 |
+
aiohttp==3.13.3
|
| 90 |
+
scikit-learn==1.7.2
|
| 91 |
+
nvidia-cutlass-dsl==4.4.0
|
| 92 |
+
setuptools==82.0.0
|
| 93 |
+
jupyter_core==5.9.1
|
| 94 |
+
asyncio==4.0.0
|
| 95 |
+
einops==0.8.2
|
| 96 |
+
pydantic==2.12.5
|
| 97 |
+
av==16.1.0
|
| 98 |
+
protobuf==6.33.5
|
| 99 |
+
evaluate==0.4.6
|
| 100 |
+
Jinja2==3.1.6
|
| 101 |
+
exceptiongroup==1.3.1
|
| 102 |
+
ipython==8.38.0
|
| 103 |
+
sentry-sdk==2.53.0
|
| 104 |
+
httpx==0.28.1
|
| 105 |
+
msgpack==1.1.2
|
| 106 |
+
platformdirs==4.9.2
|
| 107 |
+
hjson==3.1.0
|
| 108 |
+
torchvision==0.21.0
|
| 109 |
+
parso==0.8.6
|
| 110 |
+
cuda-bindings==13.1.1
|
| 111 |
+
pexpect==4.9.0
|
| 112 |
+
threadpoolctl==3.6.0
|
| 113 |
+
pyarrow==23.0.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 116 |
+
Werkzeug==3.1.5
|
| 117 |
+
fsspec==2025.10.0
|
| 118 |
+
pydantic_core==2.41.5
|
| 119 |
+
GitPython==3.1.46
|
| 120 |
+
propcache==0.4.1
|
| 121 |
+
mpmath==1.3.0
|
| 122 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 123 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 124 |
+
hf-xet==1.2.0
|
| 125 |
+
charset-normalizer==3.4.4
|
| 126 |
+
dill==0.4.0
|
| 127 |
+
datasets==4.5.0
|
| 128 |
+
py-cpuinfo==9.0.0
|
| 129 |
+
sentencepiece==0.2.1
|
| 130 |
+
latex2sympy2_extended==1.11.0
|
| 131 |
+
jupyter_client==8.8.0
|
| 132 |
+
pyzmq==27.1.0
|
| 133 |
+
numpy==2.2.6
|
| 134 |
+
gitdb==4.0.12
|
| 135 |
+
grpcio==1.78.0
|
| 136 |
+
packaging==26.0
|
| 137 |
+
requests==2.32.5
|
| 138 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 139 |
+
httpcore==1.0.9
|
| 140 |
+
traitlets==5.14.3
|
| 141 |
+
filelock==3.24.2
|
| 142 |
+
ptyprocess==0.7.0
|
| 143 |
+
pillow==12.1.1
|
| 144 |
+
torch==2.6.0
|
| 145 |
+
more-itertools==10.8.0
|
| 146 |
+
wheel==0.46.3
|
| 147 |
+
jaraco.text==4.0.0
|
| 148 |
+
zipp==3.23.0
|
| 149 |
+
autocommand==2.2.2
|
| 150 |
+
jaraco.context==6.1.0
|
| 151 |
+
jaraco.functools==4.4.0
|
| 152 |
+
tomli==2.4.0
|
| 153 |
+
backports.tarfile==1.2.0
|
| 154 |
+
importlib_metadata==8.7.1
|
| 155 |
+
platformdirs==4.4.0
|
| 156 |
+
packaging==26.0
|
wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-03-24T08:44:25.714950Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"config=configs/rl_lladav_video_reversal.yaml",
|
| 7 |
+
"experiment.current_epoch=15"
|
| 8 |
+
],
|
| 9 |
+
"program": "/data4/runqing/dLLM-RL/train/rl_lladav.py",
|
| 10 |
+
"codePath": "train/rl_lladav.py",
|
| 11 |
+
"codePathLocal": "train/rl_lladav.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
|
| 14 |
+
"commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
|
| 15 |
+
},
|
| 16 |
+
"email": "shixi_runqing@staff.weibo.com",
|
| 17 |
+
"root": "/data4/runqing/dLLM-RL",
|
| 18 |
+
"host": "b1301b38c75f",
|
| 19 |
+
"executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H800",
|
| 23 |
+
"gpu_count": 8,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "806578724864",
|
| 27 |
+
"used": "572337045504"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "2164113813504"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H800",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H800",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H800",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H800",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H800",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H800",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"name": "NVIDIA H800",
|
| 78 |
+
"memoryTotal": "85520809984",
|
| 79 |
+
"cudaCores": 16896,
|
| 80 |
+
"architecture": "Hopper",
|
| 81 |
+
"uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "NVIDIA H800",
|
| 85 |
+
"memoryTotal": "85520809984",
|
| 86 |
+
"cudaCores": 16896,
|
| 87 |
+
"architecture": "Hopper",
|
| 88 |
+
"uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
|
| 89 |
+
}
|
| 90 |
+
],
|
| 91 |
+
"cudaVersion": "12.4",
|
| 92 |
+
"writerId": "sr0a7mkap2eiuw8cpbm7o5idgj2l6pki"
|
| 93 |
+
}
|
wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_runtime":1036,"_wandb":{"runtime":1036}}
|
wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-24T16:44:25.780084766+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_w7chab5/port-1691523.txt","pid":1691523,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-03-24T16:44:25.780884873+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1691523}
|
| 3 |
+
{"time":"2026-03-24T16:44:25.780852026+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1691523-1692232-1524229333/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-03-24T16:44:25.961716708+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-03-24T16:44:25.970032581+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"eb44q2tb","id":"1(@)"}
|
| 6 |
+
{"time":"2026-03-24T16:44:30.275637457+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"eb44q2tb","id":"1(@)"}
|
| 7 |
+
{"time":"2026-03-24T16:44:35.492970552+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"pfjk4x7e02ih"}
|
| 8 |
+
{"time":"2026-03-24T17:01:46.448086485+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-03-24T17:01:46.448222721+08:00","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-03-24T17:01:46.448192503+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-03-24T17:01:46.448403899+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-03-24T17:01:46.448442769+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1691523-1692232-1524229333/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-03-24T17:01:46.588524788+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-03-24T17:01:46.588559276+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-03-24T17:01:46.588574496+08:00","level":"INFO","msg":"server is closed"}
|
wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-03-24T16:44:25.970209487+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-03-24T16:44:30.272794359+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
|
| 3 |
+
{"time":"2026-03-24T16:44:30.274781439+08:00","level":"INFO","msg":"stream: created new stream","id":"eb44q2tb"}
|
| 4 |
+
{"time":"2026-03-24T16:44:30.274957975+08:00","level":"INFO","msg":"handler: started","stream_id":"eb44q2tb"}
|
| 5 |
+
{"time":"2026-03-24T16:44:30.275605838+08:00","level":"INFO","msg":"writer: started","stream_id":"eb44q2tb"}
|
| 6 |
+
{"time":"2026-03-24T16:44:30.275607151+08:00","level":"INFO","msg":"stream: started","id":"eb44q2tb"}
|
| 7 |
+
{"time":"2026-03-24T16:44:30.275615082+08:00","level":"INFO","msg":"sender: started","stream_id":"eb44q2tb"}
|
| 8 |
+
{"time":"2026-03-24T16:44:30.27762458+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
|
| 9 |
+
{"time":"2026-03-24T16:44:35.521989789+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b3af0f9bd2bc9a1d25e7c95f703d6e7f082c92a5ddeaa4bb217a9b7cfe80832"}
|
| 10 |
+
{"time":"2026-03-24T16:44:35.522055528+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=aaee6c8449b2ae1a780ad0c20f3b6210d18b6720e515f80b6266904d04c19233"}
|
| 11 |
+
{"time":"2026-03-24T16:44:35.522099395+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189FBA88EFF948C5</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b3af0f9bd2bc9a1d25e7c95f703d6e7f082c92a5ddeaa4bb217a9b7cfe80832, Size: 12686}"}
|
| 12 |
+
{"time":"2026-03-24T16:44:35.522145559+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json</Resource><RequestId>189FBA88F07C428E</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=aaee6c8449b2ae1a780ad0c20f3b6210d18b6720e515f80b6266904d04c19233, Size: 2636}"}
|
| 13 |
+
{"time":"2026-03-24T16:44:35.524085548+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=638a1b72719105e2fa963dfad98344f6b9593e0368c367c1724abafa9f17d52a"}
|
| 14 |
+
{"time":"2026-03-24T16:44:35.52450294+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d633c46405ba4ce40f9454aeca31c83c8d300c6eae3823e88f75b255c88ec8a"}
|
| 15 |
+
{"time":"2026-03-24T16:44:35.525507802+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch</Resource><RequestId>189FBA88F08CB76A</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=638a1b72719105e2fa963dfad98344f6b9593e0368c367c1724abafa9f17d52a, Size: 12686}"}
|
| 16 |
+
{"time":"2026-03-24T16:44:35.52555316+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt</Resource><RequestId>189FBA88F01C4C5D</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d633c46405ba4ce40f9454aeca31c83c8d300c6eae3823e88f75b255c88ec8a, Size: 2966}"}
|
| 17 |
+
{"time":"2026-03-24T16:44:35.533015823+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4421e08beab520f2a67729be80eff77a1e13d1128016b44b9b31a9397524de55"}
|
| 18 |
+
{"time":"2026-03-24T16:44:35.533064379+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py</Resource><RequestId>189FBA88F09E62CB</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4421e08beab520f2a67729be80eff77a1e13d1128016b44b9b31a9397524de55, Size: 32189}"}
|
| 19 |
+
{"time":"2026-03-24T17:01:46.448220293+08:00","level":"INFO","msg":"stream: closing","id":"eb44q2tb"}
|
| 20 |
+
{"time":"2026-03-24T17:01:46.543269717+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e3cd94ab34d1b5cb38d2d62928042acf4a195aacb875e5b3ce61af7e133f5d6c"}
|
| 21 |
+
{"time":"2026-03-24T17:01:46.543380827+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log</Resource><RequestId>189FBB78FE1F23F7</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e3cd94ab34d1b5cb38d2d62928042acf4a195aacb875e5b3ce61af7e133f5d6c, Size: 2682}"}
|
| 22 |
+
{"time":"2026-03-24T17:01:46.544795606+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=37c1c8b11290535a5dccb65faaf1b5e16cbc96ec65898ab418b2ed23847cabd0"}
|
| 23 |
+
{"time":"2026-03-24T17:01:46.544900322+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json</Resource><RequestId>189FBB78FD79ECBE</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=37c1c8b11290535a5dccb65faaf1b5e16cbc96ec65898ab418b2ed23847cabd0, Size: 43}"}
|
| 24 |
+
{"time":"2026-03-24T17:01:46.546082064+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a171387400e366cd3f49b5662ca5671bbf8af4bf04a935ca313d9b14507b6cf1"}
|
| 25 |
+
{"time":"2026-03-24T17:01:46.546183734+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml</Resource><RequestId>189FBB78FDB1FAB3</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a171387400e366cd3f49b5662ca5671bbf8af4bf04a935ca313d9b14507b6cf1, Size: 7766}"}
|
| 26 |
+
{"time":"2026-03-24T17:01:46.546930214+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 27 |
+
{"time":"2026-03-24T17:01:46.587769696+08:00","level":"INFO","msg":"handler: closed","stream_id":"eb44q2tb"}
|
| 28 |
+
{"time":"2026-03-24T17:01:46.587910039+08:00","level":"INFO","msg":"sender: closed","stream_id":"eb44q2tb"}
|
| 29 |
+
{"time":"2026-03-24T17:01:46.587937144+08:00","level":"INFO","msg":"stream: closed","id":"eb44q2tb"}
|
wandb/run-20260324_164425-eb44q2tb/logs/debug.log
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
|
| 2 |
+
2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Configure stats pid to 1691523
|
| 3 |
+
2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/logs/debug.log
|
| 5 |
+
2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log
|
| 6 |
+
2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}}
|
| 9 |
+
2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-24 16:44:25,961 INFO MainThread:1691523 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-24 16:44:25,966 INFO MainThread:1691523 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-24 16:44:25,967 INFO MainThread:1691523 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-24 16:44:25,973 INFO MainThread:1691523 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-24 16:44:30,384 INFO MainThread:1691523 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-24 16:44:30,491 INFO MainThread:1691523 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-24 16:44:30,492 INFO MainThread:1691523 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'eb44q2tb', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 20, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 15, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 64, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'}
|
| 21 |
+
2026-03-24 17:01:46,448 INFO wandb-AsyncioManager-main:1691523 [service_client.py:_forward_responses():134] Reached EOF.
|
| 22 |
+
2026-03-24 17:01:46,448 INFO wandb-AsyncioManager-main:1691523 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py
ADDED
|
@@ -0,0 +1,825 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 4 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
| 5 |
+
from datetime import timedelta
|
| 6 |
+
import json
|
| 7 |
+
import logging
|
| 8 |
+
import math
|
| 9 |
+
import shutil
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Dict, List, Any
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from torch.optim import AdamW
|
| 18 |
+
from torch.utils.data import Dataset, DataLoader
|
| 19 |
+
|
| 20 |
+
from omegaconf import OmegaConf
|
| 21 |
+
import wandb
|
| 22 |
+
|
| 23 |
+
from accelerate import Accelerator
|
| 24 |
+
from accelerate.logging import get_logger
|
| 25 |
+
from accelerate.utils import set_seed
|
| 26 |
+
|
| 27 |
+
from train.utils import get_config, flatten_omega_conf, AverageMeter
|
| 28 |
+
from models.lr_schedulers import get_scheduler
|
| 29 |
+
from models.logging import set_verbosity_info, set_verbosity_error
|
| 30 |
+
|
| 31 |
+
from transformers.utils import logging as hf_logging
|
| 32 |
+
hf_logging.set_verbosity_error()
|
| 33 |
+
|
| 34 |
+
from llava.model.builder import load_pretrained_model
|
| 35 |
+
from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord
|
| 36 |
+
from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
|
| 37 |
+
from llava.conversation import conv_templates
|
| 38 |
+
|
| 39 |
+
from termcolor import cprint
|
| 40 |
+
|
| 41 |
+
logger = get_logger(__name__, log_level="INFO")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def collapse_k_unique(lst, k: int):
|
| 45 |
+
if k <= 0:
|
| 46 |
+
raise ValueError("k must be > 0")
|
| 47 |
+
uniq = sorted(set(lst))
|
| 48 |
+
mapping = {}
|
| 49 |
+
n = len(uniq)
|
| 50 |
+
for idx, val in enumerate(uniq):
|
| 51 |
+
group = idx // k
|
| 52 |
+
end_idx = min((group + 1) * k - 1, n - 1)
|
| 53 |
+
rep = uniq[end_idx]
|
| 54 |
+
mapping[val] = rep
|
| 55 |
+
return [mapping[x] for x in lst]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
|
| 59 |
+
import copy
|
| 60 |
+
conv = copy.deepcopy(conv_templates["llava_llada"])
|
| 61 |
+
conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
|
| 62 |
+
conv.append_message(conv.roles[1], response_text)
|
| 63 |
+
return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
|
| 67 |
+
conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
|
| 68 |
+
ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
|
| 69 |
+
return ids.squeeze(0)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
|
| 73 |
+
from PIL import Image
|
| 74 |
+
from types import SimpleNamespace
|
| 75 |
+
|
| 76 |
+
if isinstance(model_cfg, dict):
|
| 77 |
+
model_cfg = SimpleNamespace(**model_cfg)
|
| 78 |
+
|
| 79 |
+
if media_type == "image":
|
| 80 |
+
image = Image.open(abs_path).convert("RGB")
|
| 81 |
+
if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
|
| 82 |
+
setattr(model_cfg, "image_aspect_ratio", "pad")
|
| 83 |
+
if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
|
| 84 |
+
setattr(model_cfg, "mm_patch_merge_type", "flat")
|
| 85 |
+
|
| 86 |
+
image_tensor_list = process_images([image], image_processor, model_cfg)
|
| 87 |
+
image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
|
| 88 |
+
image_sizes = [image.size]
|
| 89 |
+
return image_tensor_list, image_sizes, None
|
| 90 |
+
|
| 91 |
+
elif media_type == "video":
|
| 92 |
+
if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
|
| 93 |
+
if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16)
|
| 94 |
+
if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True)
|
| 95 |
+
|
| 96 |
+
video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
|
| 97 |
+
image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
|
| 98 |
+
image_tensor = image_tensor.to(dtype=dtype, device=device)
|
| 99 |
+
|
| 100 |
+
if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
|
| 101 |
+
v_size = (video_frames.shape[2], video_frames.shape[1])
|
| 102 |
+
else:
|
| 103 |
+
v_size = (0, 0)
|
| 104 |
+
|
| 105 |
+
return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
|
| 109 |
+
out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
|
| 110 |
+
out[-x.numel():] = x
|
| 111 |
+
return out
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
|
| 115 |
+
if x.size(0) == L:
|
| 116 |
+
return x
|
| 117 |
+
out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
|
| 118 |
+
out[-x.size(0):] = x
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
|
| 123 |
+
p = sample.get(f"{media_type}_abs_path")
|
| 124 |
+
if p is None:
|
| 125 |
+
p = sample.get(media_type) or sample.get(f"{media_type}_path")
|
| 126 |
+
if p is None:
|
| 127 |
+
return None
|
| 128 |
+
if not os.path.isabs(p) and dataset_root:
|
| 129 |
+
p = os.path.join(dataset_root, p)
|
| 130 |
+
try:
|
| 131 |
+
return os.path.abspath(p)
|
| 132 |
+
except Exception:
|
| 133 |
+
return p
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class RLVariantDatasetLLADAV(Dataset):
|
| 137 |
+
def __init__(self, samples: List[dict], config):
|
| 138 |
+
super().__init__()
|
| 139 |
+
self.samples = samples
|
| 140 |
+
self.config = config
|
| 141 |
+
self.variants: List[dict] = []
|
| 142 |
+
|
| 143 |
+
# logp_old_tok mapping: idx -> tensor
|
| 144 |
+
self.logp_old_tok: Dict[int, torch.Tensor] = {}
|
| 145 |
+
|
| 146 |
+
self._build_variants()
|
| 147 |
+
|
| 148 |
+
def _build_variants(self):
|
| 149 |
+
sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
|
| 150 |
+
question_prompt = "\nOnly give the best option."
|
| 151 |
+
multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
|
| 152 |
+
|
| 153 |
+
rng = np.random.default_rng(seed=self.config.training.seed or 42)
|
| 154 |
+
idx_counter = 0
|
| 155 |
+
|
| 156 |
+
for sid, s in enumerate(self.samples):
|
| 157 |
+
# Rollout produces "extracted_output" or "response", fall back if needed
|
| 158 |
+
response_text = s.get("response", s.get("extracted_output", ""))
|
| 159 |
+
if isinstance(response_text, list) and len(response_text) > 0:
|
| 160 |
+
response_text = response_text[0] # Just in case it's nested
|
| 161 |
+
|
| 162 |
+
# If options exist (video multiple choice), construct full prompt
|
| 163 |
+
if "options" in s:
|
| 164 |
+
prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt
|
| 165 |
+
else:
|
| 166 |
+
prompt_text = s["prompt"]
|
| 167 |
+
|
| 168 |
+
step_map = s.get("step_map", [])
|
| 169 |
+
|
| 170 |
+
if self.config.training.method == "TraceRL":
|
| 171 |
+
order_list = collapse_k_unique(step_map, self.config.training.shrink)
|
| 172 |
+
uniq_steps = sorted(set(order_list))
|
| 173 |
+
for st in uniq_steps:
|
| 174 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None))
|
| 175 |
+
idx_counter += 1
|
| 176 |
+
|
| 177 |
+
elif self.config.training.method == "random_masking":
|
| 178 |
+
for _ in range(self.config.training.mask_times_per_sample):
|
| 179 |
+
t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
|
| 180 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 181 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed))
|
| 182 |
+
idx_counter += 1
|
| 183 |
+
|
| 184 |
+
elif self.config.training.method == "coupled":
|
| 185 |
+
for _ in range(self.config.training.mask_times_per_sample):
|
| 186 |
+
t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
|
| 187 |
+
mask_seed = int(rng.integers(0, 2**31 - 1))
|
| 188 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed))
|
| 189 |
+
idx_counter += 1
|
| 190 |
+
self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed))
|
| 191 |
+
idx_counter += 1
|
| 192 |
+
else:
|
| 193 |
+
raise ValueError(f"Unknown RL method: {self.config.training.method}")
|
| 194 |
+
|
| 195 |
+
def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed):
|
| 196 |
+
return dict(
|
| 197 |
+
idx=idx,
|
| 198 |
+
sample_id=sid,
|
| 199 |
+
prompt=prompt,
|
| 200 |
+
response=response,
|
| 201 |
+
image_abs_path=s.get("image_abs_path", None),
|
| 202 |
+
video_abs_path=s.get("video_abs_path", None),
|
| 203 |
+
reward=s.get("reward", 0.0),
|
| 204 |
+
var_type=var_type,
|
| 205 |
+
var_param=var_param,
|
| 206 |
+
step_map=step_map,
|
| 207 |
+
mask_seed=mask_seed,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
def __len__(self):
|
| 211 |
+
return len(self.variants)
|
| 212 |
+
|
| 213 |
+
def __getitem__(self, idx):
|
| 214 |
+
return self.variants[idx]
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
|
| 218 |
+
out: Dict[str, List[Any]] = {}
|
| 219 |
+
keys = batch[0].keys()
|
| 220 |
+
for k in keys:
|
| 221 |
+
out[k] = [b[k] for b in batch]
|
| 222 |
+
return out
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
@torch.no_grad()
|
| 226 |
+
def prepare_batch_noisy_embeds_rl(
|
| 227 |
+
accelerator,
|
| 228 |
+
model,
|
| 229 |
+
tokenizer,
|
| 230 |
+
image_processor,
|
| 231 |
+
batch: Dict[str, List[Any]],
|
| 232 |
+
mask_id: int,
|
| 233 |
+
dataset_media_root: str | None,
|
| 234 |
+
) -> tuple | None:
|
| 235 |
+
|
| 236 |
+
device = accelerator.device
|
| 237 |
+
bsz = len(batch["idx"])
|
| 238 |
+
|
| 239 |
+
group: Dict[int, List[int]] = {}
|
| 240 |
+
for i, sid in enumerate(batch["sample_id"]):
|
| 241 |
+
group.setdefault(int(sid), []).append(i)
|
| 242 |
+
|
| 243 |
+
sample_cache: Dict[int, Dict[str, Any]] = {}
|
| 244 |
+
for sid, pos_list in group.items():
|
| 245 |
+
first_pos = pos_list[0]
|
| 246 |
+
prompt_text = batch["prompt"][first_pos]
|
| 247 |
+
response_text = batch["response"][first_pos]
|
| 248 |
+
|
| 249 |
+
sample_stub = {
|
| 250 |
+
"image_abs_path": batch.get("image_abs_path", [None])[first_pos],
|
| 251 |
+
"video_abs_path": batch.get("video_abs_path", [None])[first_pos],
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
media_type = None
|
| 255 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
|
| 256 |
+
if p_abs is not None:
|
| 257 |
+
media_type = "video"
|
| 258 |
+
else:
|
| 259 |
+
p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
|
| 260 |
+
if p_abs is not None:
|
| 261 |
+
media_type = "image"
|
| 262 |
+
|
| 263 |
+
img_list, image_sizes, modalities = None, None, []
|
| 264 |
+
if p_abs is not None:
|
| 265 |
+
img_list, image_sizes, video_stats = load_and_process_media(
|
| 266 |
+
p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
|
| 267 |
+
)
|
| 268 |
+
modalities = [media_type]
|
| 269 |
+
|
| 270 |
+
if media_type == "video" and getattr(model.config, "add_time_instruction", False):
|
| 271 |
+
video_time, frame_time, num_frames = video_stats
|
| 272 |
+
time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
|
| 273 |
+
clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
|
| 274 |
+
prompt_text = f"{time_instruction}\n{clean_prompt}"
|
| 275 |
+
|
| 276 |
+
input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
|
| 277 |
+
end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
|
| 278 |
+
seq_len = len(end_header_seq)
|
| 279 |
+
pos_eh = -1
|
| 280 |
+
if seq_len > 0 and input_ids_text.numel() >= seq_len:
|
| 281 |
+
ids_list = input_ids_text.tolist()
|
| 282 |
+
for i in range(len(ids_list) - seq_len, -1, -1):
|
| 283 |
+
if ids_list[i:i + seq_len] == end_header_seq:
|
| 284 |
+
pos_eh = i + seq_len - 1
|
| 285 |
+
break
|
| 286 |
+
assert pos_eh != -1, " <|end_header_id|> not found"
|
| 287 |
+
start_pos = pos_eh + 1
|
| 288 |
+
|
| 289 |
+
labels_text = input_ids_text.clone()
|
| 290 |
+
labels_text[:start_pos] = -100
|
| 291 |
+
|
| 292 |
+
(_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
|
| 293 |
+
input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
|
| 294 |
+
)
|
| 295 |
+
clean_embeds = clean_embeds.squeeze(0)
|
| 296 |
+
new_labels = new_labels.squeeze(0)
|
| 297 |
+
|
| 298 |
+
eot_id = 126348
|
| 299 |
+
eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
|
| 300 |
+
if eot_indices.numel() > 0:
|
| 301 |
+
last_eot_pos = eot_indices[-1, 0]
|
| 302 |
+
new_labels[last_eot_pos:] = -100
|
| 303 |
+
asst_mask = (new_labels != -100)
|
| 304 |
+
asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1)
|
| 305 |
+
|
| 306 |
+
sample_cache[sid] = dict(
|
| 307 |
+
clean_embeds=clean_embeds,
|
| 308 |
+
new_labels=new_labels,
|
| 309 |
+
asst_idx=asst_idx,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)]
|
| 313 |
+
Lmax = max(lengths)
|
| 314 |
+
mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
|
| 315 |
+
|
| 316 |
+
noisy_list, labels_list, p_mask_list, attn_list = [], [], [], []
|
| 317 |
+
reward_list, valid_idxs = [], []
|
| 318 |
+
|
| 319 |
+
for i in range(bsz):
|
| 320 |
+
sid = int(batch["sample_id"][i])
|
| 321 |
+
var_type = batch["var_type"][i]
|
| 322 |
+
var_param = batch["var_param"][i]
|
| 323 |
+
|
| 324 |
+
clean_embeds = sample_cache[sid]["clean_embeds"].clone()
|
| 325 |
+
new_labels = sample_cache[sid]["new_labels"].clone()
|
| 326 |
+
asst_idx = sample_cache[sid]["asst_idx"]
|
| 327 |
+
|
| 328 |
+
L = clean_embeds.size(0)
|
| 329 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 330 |
+
M = asst_idx.numel()
|
| 331 |
+
|
| 332 |
+
if M == 0:
|
| 333 |
+
continue # skip badly formatted inputs
|
| 334 |
+
|
| 335 |
+
if var_type == "TraceRL":
|
| 336 |
+
step_map = batch["step_map"][i]
|
| 337 |
+
order = torch.as_tensor(step_map, device=device)
|
| 338 |
+
m_len = min(M, order.numel())
|
| 339 |
+
order_full = torch.full((L,), -1, dtype=torch.long, device=device)
|
| 340 |
+
order_full[asst_idx[:m_len]] = order[:m_len]
|
| 341 |
+
step_val = int(var_param)
|
| 342 |
+
|
| 343 |
+
tgt_mask = (order_full == step_val)
|
| 344 |
+
pmask = tgt_mask
|
| 345 |
+
mask_pos = (order_full >= step_val)
|
| 346 |
+
|
| 347 |
+
noisy_embeds = clean_embeds.clone()
|
| 348 |
+
noisy_embeds[mask_pos] = mask_embed
|
| 349 |
+
|
| 350 |
+
elif var_type == "random_masking" or var_type.startswith("coupled"):
|
| 351 |
+
if M == 0:
|
| 352 |
+
# M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward
|
| 353 |
+
noisy_embeds = clean_embeds.clone()
|
| 354 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 355 |
+
else:
|
| 356 |
+
t = float(var_param)
|
| 357 |
+
# 更安全的 seed 获取方式
|
| 358 |
+
mask_seed = batch.get("mask_seed", [None] * bsz)[i]
|
| 359 |
+
if mask_seed is None:
|
| 360 |
+
mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
|
| 361 |
+
|
| 362 |
+
gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
|
| 363 |
+
rnd_cpu = torch.rand(M, generator=gen) < t
|
| 364 |
+
|
| 365 |
+
# 如果是 coupled_comp (对比样本),需要取反
|
| 366 |
+
if var_type == "coupled_comp":
|
| 367 |
+
rnd_cpu = ~rnd_cpu
|
| 368 |
+
|
| 369 |
+
rnd = rnd_cpu.to(device)
|
| 370 |
+
|
| 371 |
+
# 🌟 上帝之手:强行兜底逻辑
|
| 372 |
+
# 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁
|
| 373 |
+
if not rnd.any():
|
| 374 |
+
# 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反,
|
| 375 |
+
# 除非 coupled_rand 全是 True,否则 comp 不可能全 False。
|
| 376 |
+
# 这里统一下一个兜底,非常安全。
|
| 377 |
+
rand_pos = torch.randint(0, M, (1,), device=device)
|
| 378 |
+
rnd[rand_pos] = True
|
| 379 |
+
|
| 380 |
+
pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 381 |
+
pmask[asst_idx] = rnd
|
| 382 |
+
|
| 383 |
+
noisy_embeds = clean_embeds.clone()
|
| 384 |
+
noisy_embeds[pmask] = mask_embed
|
| 385 |
+
|
| 386 |
+
# elif var_type == "random_masking" or var_type.startswith("coupled"):
|
| 387 |
+
# t = float(var_param)
|
| 388 |
+
# mask_seed = batch["mask_seed"][i]
|
| 389 |
+
# gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
|
| 390 |
+
# rnd_cpu = torch.rand(M, generator=gen) < t
|
| 391 |
+
|
| 392 |
+
# if var_type == "coupled_comp":
|
| 393 |
+
# rnd_cpu = ~rnd_cpu
|
| 394 |
+
|
| 395 |
+
# pmask = torch.zeros(L, dtype=torch.bool, device=device)
|
| 396 |
+
# pmask[asst_idx] = rnd_cpu.to(device)
|
| 397 |
+
|
| 398 |
+
# noisy_embeds = clean_embeds.clone()
|
| 399 |
+
# noisy_embeds[pmask] = mask_embed
|
| 400 |
+
|
| 401 |
+
if not pmask.any():
|
| 402 |
+
print('************not pmask.any()**********')
|
| 403 |
+
continue
|
| 404 |
+
|
| 405 |
+
noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
|
| 406 |
+
labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
|
| 407 |
+
p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
|
| 408 |
+
attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
|
| 409 |
+
attn[-L:] = True
|
| 410 |
+
attn_list.append(attn)
|
| 411 |
+
reward_list.append(float(batch["reward"][i]))
|
| 412 |
+
valid_idxs.append(batch["idx"][i])
|
| 413 |
+
|
| 414 |
+
if len(noisy_list) == 0:
|
| 415 |
+
return None
|
| 416 |
+
|
| 417 |
+
noisy_batch = torch.stack(noisy_list, dim=0)
|
| 418 |
+
labels_batch = torch.stack(labels_list, dim=0)
|
| 419 |
+
p_mask_batch = torch.stack(p_mask_list, dim=0)
|
| 420 |
+
attn_batch = torch.stack(attn_list, dim=0)
|
| 421 |
+
reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32)
|
| 422 |
+
|
| 423 |
+
return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
@torch.no_grad()
|
| 427 |
+
def compute_logp_old_tok_parallel(
|
| 428 |
+
accelerator,
|
| 429 |
+
model,
|
| 430 |
+
tokenizer,
|
| 431 |
+
image_processor,
|
| 432 |
+
dataset,
|
| 433 |
+
dl,
|
| 434 |
+
mask_id,
|
| 435 |
+
config):
|
| 436 |
+
|
| 437 |
+
model.eval()
|
| 438 |
+
logger.info("Computing reference log probabilities (old_lp) for all variants...")
|
| 439 |
+
|
| 440 |
+
for batch in dl:
|
| 441 |
+
out = prepare_batch_noisy_embeds_rl(
|
| 442 |
+
accelerator=accelerator,
|
| 443 |
+
model=model,
|
| 444 |
+
tokenizer=tokenizer,
|
| 445 |
+
image_processor=image_processor,
|
| 446 |
+
batch=batch,
|
| 447 |
+
mask_id=mask_id,
|
| 448 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 449 |
+
)
|
| 450 |
+
if out is None:
|
| 451 |
+
continue
|
| 452 |
+
|
| 453 |
+
noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out
|
| 454 |
+
|
| 455 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True)
|
| 456 |
+
logits = model.lm_head(outputs.last_hidden_state).float()
|
| 457 |
+
|
| 458 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 459 |
+
safe_labels = labels_batch.clone()
|
| 460 |
+
safe_labels[labels_batch == -100] = 0
|
| 461 |
+
tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
|
| 462 |
+
|
| 463 |
+
for i, idx in enumerate(valid_idxs):
|
| 464 |
+
dataset.logp_old_tok[idx] = tok_lp[i].float().cpu()
|
| 465 |
+
|
| 466 |
+
accelerator.wait_for_everyone()
|
| 467 |
+
model.train()
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
def save_checkpoint(model, tokenizer, config, accelerator, name):
|
| 471 |
+
output_dir = Path(config.experiment.project)
|
| 472 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 473 |
+
|
| 474 |
+
checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
|
| 475 |
+
|
| 476 |
+
if accelerator.is_main_process and checkpoints_total_limit is not None:
|
| 477 |
+
ckpts = sorted(
|
| 478 |
+
[d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
|
| 479 |
+
key=lambda p: int(p.name.split("-")[1]),
|
| 480 |
+
)
|
| 481 |
+
if len(ckpts) >= checkpoints_total_limit:
|
| 482 |
+
to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
|
| 483 |
+
logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
|
| 484 |
+
for p in to_remove:
|
| 485 |
+
shutil.rmtree(p, ignore_errors=True)
|
| 486 |
+
|
| 487 |
+
save_base = output_dir / "ckpt"
|
| 488 |
+
save_base.mkdir(exist_ok=True)
|
| 489 |
+
|
| 490 |
+
model_to_save = accelerator.unwrap_model(model)
|
| 491 |
+
state_dict = accelerator.get_state_dict(model)
|
| 492 |
+
|
| 493 |
+
if accelerator.is_main_process:
|
| 494 |
+
model_to_save.save_pretrained(
|
| 495 |
+
save_base / name,
|
| 496 |
+
save_function=accelerator.save,
|
| 497 |
+
state_dict=state_dict,
|
| 498 |
+
safe_serialization=True,
|
| 499 |
+
)
|
| 500 |
+
tokenizer.save_pretrained(str(save_base / name))
|
| 501 |
+
|
| 502 |
+
metadata = {
|
| 503 |
+
"save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 504 |
+
}
|
| 505 |
+
with (save_base / "metadata.json").open("w") as f:
|
| 506 |
+
json.dump(metadata, f, indent=2)
|
| 507 |
+
|
| 508 |
+
logger.info(f"Saved model + tokenizer to {save_base / name}")
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def main():
|
| 512 |
+
config = get_config()
|
| 513 |
+
|
| 514 |
+
project_name = config.experiment.project
|
| 515 |
+
if config.experiment.current_epoch == 1:
|
| 516 |
+
pretrained_model = config.model.pretrained_model
|
| 517 |
+
else:
|
| 518 |
+
pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name
|
| 519 |
+
|
| 520 |
+
if config.training.enable_tf32:
|
| 521 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 522 |
+
torch.backends.cudnn.benchmark = True
|
| 523 |
+
torch.backends.cudnn.deterministic = False
|
| 524 |
+
|
| 525 |
+
config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
|
| 526 |
+
accelerator = Accelerator(
|
| 527 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 528 |
+
mixed_precision=config.training.mixed_precision,
|
| 529 |
+
log_with='wandb',
|
| 530 |
+
project_dir=config.experiment.logging_dir,
|
| 531 |
+
split_batches=True,
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
logging.basicConfig(
|
| 535 |
+
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
| 536 |
+
datefmt="%m/%d/%Y %H:%M:%S",
|
| 537 |
+
level=logging.INFO,
|
| 538 |
+
)
|
| 539 |
+
logger.info(accelerator.state, main_process_only=False)
|
| 540 |
+
if accelerator.is_local_main_process:
|
| 541 |
+
set_verbosity_info()
|
| 542 |
+
else:
|
| 543 |
+
set_verbosity_error()
|
| 544 |
+
|
| 545 |
+
if accelerator.is_main_process:
|
| 546 |
+
resume_wandb_run = config.wandb.resume
|
| 547 |
+
run_id = config.wandb.get("run_id", None)
|
| 548 |
+
if run_id is None:
|
| 549 |
+
resume_wandb_run = False
|
| 550 |
+
run_id = wandb.util.generate_id()
|
| 551 |
+
config.wandb.run_id = run_id
|
| 552 |
+
|
| 553 |
+
wandb_init_kwargs = dict(
|
| 554 |
+
name=config.experiment.project,
|
| 555 |
+
id=run_id,
|
| 556 |
+
resume=resume_wandb_run,
|
| 557 |
+
entity=config.wandb.get("entity", None),
|
| 558 |
+
config_exclude_keys=[],
|
| 559 |
+
)
|
| 560 |
+
wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
|
| 561 |
+
wandb_config.pop("experiment.resume_from_checkpoint", None)
|
| 562 |
+
|
| 563 |
+
accelerator.init_trackers(
|
| 564 |
+
config.experiment.project,
|
| 565 |
+
config=wandb_config,
|
| 566 |
+
init_kwargs={"wandb": wandb_init_kwargs},
|
| 567 |
+
)
|
| 568 |
+
|
| 569 |
+
if accelerator.is_main_process:
|
| 570 |
+
os.makedirs(config.experiment.project, exist_ok=True)
|
| 571 |
+
config_path = Path(config.experiment.project) / "config.yaml"
|
| 572 |
+
logging.info(f"Saving config to {config_path}")
|
| 573 |
+
OmegaConf.save(config, config_path)
|
| 574 |
+
|
| 575 |
+
if config.training.seed is not None:
|
| 576 |
+
set_seed(config.training.seed)
|
| 577 |
+
|
| 578 |
+
logger.info("Loading LLaDA-V model and tokenizer")
|
| 579 |
+
|
| 580 |
+
tokenizer, model, image_processor, _ = load_pretrained_model(
|
| 581 |
+
pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
|
| 582 |
+
)
|
| 583 |
+
model = model.to(accelerator.device)
|
| 584 |
+
|
| 585 |
+
special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
|
| 586 |
+
num_new = tokenizer.add_special_tokens(special_tokens)
|
| 587 |
+
if num_new > 0:
|
| 588 |
+
model.resize_token_embeddings(len(tokenizer))
|
| 589 |
+
if tokenizer.pad_token is None:
|
| 590 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 591 |
+
|
| 592 |
+
if getattr(config.training, "gradient_checkpointing_enable", False):
|
| 593 |
+
try:
|
| 594 |
+
model.enable_input_require_grads()
|
| 595 |
+
except Exception:
|
| 596 |
+
pass
|
| 597 |
+
try:
|
| 598 |
+
model.gradient_checkpointing_enable()
|
| 599 |
+
except Exception:
|
| 600 |
+
pass
|
| 601 |
+
|
| 602 |
+
mask_id = 126336
|
| 603 |
+
|
| 604 |
+
optimizer_config = config.optimizer.params
|
| 605 |
+
no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
|
| 606 |
+
optimizer_grouped_parameters = [
|
| 607 |
+
{
|
| 608 |
+
"params": [p for n, p in model.named_parameters() if
|
| 609 |
+
p.requires_grad and not any(nd in n for nd in no_decay)],
|
| 610 |
+
"weight_decay": optimizer_config.weight_decay,
|
| 611 |
+
},
|
| 612 |
+
{
|
| 613 |
+
"params": [p for n, p in model.named_parameters() if
|
| 614 |
+
p.requires_grad and any(nd in n for nd in no_decay)],
|
| 615 |
+
"weight_decay": 0.0,
|
| 616 |
+
},
|
| 617 |
+
]
|
| 618 |
+
|
| 619 |
+
optimizer_type = config.optimizer.name
|
| 620 |
+
if optimizer_type == "adamw":
|
| 621 |
+
optimizer = AdamW(
|
| 622 |
+
optimizer_grouped_parameters,
|
| 623 |
+
lr=optimizer_config.learning_rate,
|
| 624 |
+
betas=(optimizer_config.beta1, optimizer_config.beta2),
|
| 625 |
+
weight_decay=optimizer_config.weight_decay,
|
| 626 |
+
eps=optimizer_config.epsilon,
|
| 627 |
+
)
|
| 628 |
+
else:
|
| 629 |
+
raise ValueError(f"Optimizer {optimizer_type} not supported")
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
logger.info("Preprocessing Data")
|
| 633 |
+
|
| 634 |
+
with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f:
|
| 635 |
+
dataset_load = json.load(f)
|
| 636 |
+
|
| 637 |
+
dataset_lm = RLVariantDatasetLLADAV(dataset_load, config)
|
| 638 |
+
|
| 639 |
+
logger.info("Preparing model, optimizer and dataloaders")
|
| 640 |
+
|
| 641 |
+
total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
|
| 642 |
+
num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm)
|
| 643 |
+
num_train_epochs = config.training.num_train_epochs
|
| 644 |
+
max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
|
| 645 |
+
|
| 646 |
+
lr_scheduler = get_scheduler(
|
| 647 |
+
config.lr_scheduler.scheduler,
|
| 648 |
+
optimizer=optimizer,
|
| 649 |
+
num_training_steps=max_train_steps,
|
| 650 |
+
num_warmup_steps=config.lr_scheduler.params.warmup_steps,
|
| 651 |
+
min_lr_scale=config.lr_scheduler.params.min_lr_scale
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
train_dataloader_lm = DataLoader(
|
| 655 |
+
dataset_lm,
|
| 656 |
+
batch_size=config.training.batch_size_lm,
|
| 657 |
+
sampler=None,
|
| 658 |
+
collate_fn=collate_variants,
|
| 659 |
+
num_workers=0,
|
| 660 |
+
shuffle=False
|
| 661 |
+
)
|
| 662 |
+
|
| 663 |
+
model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare(
|
| 664 |
+
model, optimizer, lr_scheduler, train_dataloader_lm
|
| 665 |
+
)
|
| 666 |
+
|
| 667 |
+
logger.info("***** Running inference *****")
|
| 668 |
+
compute_logp_old_tok_parallel(
|
| 669 |
+
accelerator,
|
| 670 |
+
model,
|
| 671 |
+
tokenizer,
|
| 672 |
+
image_processor,
|
| 673 |
+
dataset_lm,
|
| 674 |
+
train_dataloader_lm,
|
| 675 |
+
mask_id=mask_id,
|
| 676 |
+
config=config
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
logger.info("***** Running training *****")
|
| 680 |
+
logger.info(f" Num variants = {len(dataset_lm)}")
|
| 681 |
+
logger.info(f" Num training steps = {max_train_steps}")
|
| 682 |
+
logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
|
| 683 |
+
logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
|
| 684 |
+
logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
|
| 685 |
+
|
| 686 |
+
first_epoch = 0
|
| 687 |
+
data_time_m = AverageMeter()
|
| 688 |
+
end = time.time()
|
| 689 |
+
grad_accum = config.training.gradient_accumulation_steps
|
| 690 |
+
|
| 691 |
+
def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok):
|
| 692 |
+
outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
|
| 693 |
+
logits = model.lm_head(outputs.last_hidden_state).float()
|
| 694 |
+
|
| 695 |
+
B, T, V = logits.shape
|
| 696 |
+
adv = torch.as_tensor(adv, device=noisy_embeds.device).detach()
|
| 697 |
+
|
| 698 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 699 |
+
safe_labels = labels.clone()
|
| 700 |
+
safe_labels[labels == -100] = 0
|
| 701 |
+
logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
|
| 702 |
+
|
| 703 |
+
ratio = torch.exp(logp_new_tok - logp_old_tok)
|
| 704 |
+
clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps)
|
| 705 |
+
adv_tok = adv.unsqueeze(1)
|
| 706 |
+
|
| 707 |
+
surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok)
|
| 708 |
+
surrogate_tok = surrogate_tok * p_mask_lm
|
| 709 |
+
num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1)
|
| 710 |
+
surrogate_tok = surrogate_tok.sum(dim=1) / num_mask
|
| 711 |
+
policy_loss = - (surrogate_tok.sum() / B)
|
| 712 |
+
|
| 713 |
+
# KL penalty
|
| 714 |
+
kl_loss = torch.tensor(0.0, device=policy_loss.device)
|
| 715 |
+
if config.training.beta > 0:
|
| 716 |
+
kl_seq = logp_new_tok - logp_old_tok
|
| 717 |
+
if config.training.use_kl_estimator_k3:
|
| 718 |
+
kl_seq = (-kl_seq).exp() - 1.0 + kl_seq
|
| 719 |
+
kl_seq = (kl_seq * p_mask_lm).sum(dim=1)
|
| 720 |
+
kl_loss = config.training.beta * kl_seq.sum() / B
|
| 721 |
+
total_loss = policy_loss + kl_loss
|
| 722 |
+
else:
|
| 723 |
+
total_loss = policy_loss
|
| 724 |
+
|
| 725 |
+
return total_loss
|
| 726 |
+
|
| 727 |
+
from tqdm.auto import tqdm
|
| 728 |
+
|
| 729 |
+
cprint("start rl training...", "red")
|
| 730 |
+
for epoch in range(first_epoch, num_train_epochs):
|
| 731 |
+
model.train()
|
| 732 |
+
progress_bar = tqdm(
|
| 733 |
+
train_dataloader_lm,
|
| 734 |
+
desc=f"Epoch {epoch+1}/{num_train_epochs}",
|
| 735 |
+
disable=not accelerator.is_local_main_process,
|
| 736 |
+
dynamic_ncols=True,
|
| 737 |
+
leave=True
|
| 738 |
+
)
|
| 739 |
+
|
| 740 |
+
step = 0
|
| 741 |
+
for step, batch in enumerate(progress_bar, start=1):
|
| 742 |
+
data_time_m.update(time.time() - end)
|
| 743 |
+
|
| 744 |
+
out = prepare_batch_noisy_embeds_rl(
|
| 745 |
+
accelerator=accelerator,
|
| 746 |
+
model=model,
|
| 747 |
+
tokenizer=tokenizer,
|
| 748 |
+
image_processor=image_processor,
|
| 749 |
+
batch=batch,
|
| 750 |
+
mask_id=mask_id,
|
| 751 |
+
dataset_media_root=config.dataset.get("image_root", None),
|
| 752 |
+
)
|
| 753 |
+
|
| 754 |
+
if out is None:
|
| 755 |
+
continue
|
| 756 |
+
|
| 757 |
+
noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out
|
| 758 |
+
|
| 759 |
+
# Align old log probabilities shape with current noisy_embeds
|
| 760 |
+
L_current = noisy_embeds.shape[1]
|
| 761 |
+
old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device)
|
| 762 |
+
for i, idx in enumerate(valid_idxs):
|
| 763 |
+
old_t = dataset_lm.logp_old_tok[idx]
|
| 764 |
+
l_min = min(L_current, old_t.shape[0])
|
| 765 |
+
old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device)
|
| 766 |
+
|
| 767 |
+
loss_lm = forward_process(
|
| 768 |
+
noisy_embeds=noisy_embeds,
|
| 769 |
+
labels=labels,
|
| 770 |
+
p_mask_lm=p_mask_lm,
|
| 771 |
+
attn_mask=attn_mask,
|
| 772 |
+
adv=rewards,
|
| 773 |
+
logp_old_tok=old_lp_batch
|
| 774 |
+
)
|
| 775 |
+
|
| 776 |
+
loss_lm = loss_lm / grad_accum
|
| 777 |
+
accelerator.backward(loss_lm)
|
| 778 |
+
|
| 779 |
+
if step % grad_accum == 0:
|
| 780 |
+
current_lr = optimizer.param_groups[0]['lr']
|
| 781 |
+
# current_global_step = global_step_offset + (step // grad_accum)
|
| 782 |
+
|
| 783 |
+
accelerator.log({
|
| 784 |
+
"train/loss": loss_lm.item() * grad_accum,
|
| 785 |
+
"train/learning_rate": current_lr,
|
| 786 |
+
# "train/iteration": config.experiment.current_epoch,
|
| 787 |
+
}, step=(step // grad_accum))
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
if config.training.max_grad_norm is not None:
|
| 791 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 792 |
+
optimizer.step()
|
| 793 |
+
lr_scheduler.step()
|
| 794 |
+
optimizer.zero_grad(set_to_none=True)
|
| 795 |
+
torch.cuda.empty_cache()
|
| 796 |
+
|
| 797 |
+
# Catch remaining gradients
|
| 798 |
+
if step % grad_accum != 0:
|
| 799 |
+
current_lr = optimizer.param_groups[0]['lr']
|
| 800 |
+
# current_global_step = global_step_offset + (step // grad_accum)
|
| 801 |
+
|
| 802 |
+
accelerator.log({
|
| 803 |
+
"train/loss": loss_lm.item() * grad_accum,
|
| 804 |
+
"train/learning_rate": current_lr,
|
| 805 |
+
# "train/iteration": config.experiment.current_epoch,
|
| 806 |
+
}, step=(step // grad_accum))
|
| 807 |
+
|
| 808 |
+
if config.training.max_grad_norm is not None:
|
| 809 |
+
accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
|
| 810 |
+
optimizer.step()
|
| 811 |
+
lr_scheduler.step()
|
| 812 |
+
optimizer.zero_grad(set_to_none=True)
|
| 813 |
+
torch.cuda.empty_cache()
|
| 814 |
+
|
| 815 |
+
accelerator.wait_for_everyone()
|
| 816 |
+
|
| 817 |
+
save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
|
| 818 |
+
if config.experiment.current_epoch % config.experiment.save_every == 0:
|
| 819 |
+
save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}")
|
| 820 |
+
|
| 821 |
+
accelerator.end_training()
|
| 822 |
+
|
| 823 |
+
|
| 824 |
+
if __name__ == "__main__":
|
| 825 |
+
main()
|
wandb/run-20260325_141344-o1lsxhwu/files/config.yaml
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.0
|
| 4 |
+
code_path: code/train/rl_lladav.py
|
| 5 |
+
e:
|
| 6 |
+
l1zvo1vvzkyq01b0wblads1a5x42yuiv:
|
| 7 |
+
args:
|
| 8 |
+
- config=configs/rl_lladav_video_reversal.yaml
|
| 9 |
+
- experiment.current_epoch=27
|
| 10 |
+
codePath: train/rl_lladav.py
|
| 11 |
+
codePathLocal: train/rl_lladav.py
|
| 12 |
+
cpu_count: 96
|
| 13 |
+
cpu_count_logical: 192
|
| 14 |
+
cudaVersion: "12.4"
|
| 15 |
+
disk:
|
| 16 |
+
/:
|
| 17 |
+
total: "806578724864"
|
| 18 |
+
used: "572884140032"
|
| 19 |
+
email: shixi_runqing@staff.weibo.com
|
| 20 |
+
executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10
|
| 21 |
+
git:
|
| 22 |
+
commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
|
| 23 |
+
remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
|
| 24 |
+
gpu: NVIDIA H800
|
| 25 |
+
gpu_count: 8
|
| 26 |
+
gpu_nvidia:
|
| 27 |
+
- architecture: Hopper
|
| 28 |
+
cudaCores: 16896
|
| 29 |
+
memoryTotal: "85520809984"
|
| 30 |
+
name: NVIDIA H800
|
| 31 |
+
uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
|
| 32 |
+
- architecture: Hopper
|
| 33 |
+
cudaCores: 16896
|
| 34 |
+
memoryTotal: "85520809984"
|
| 35 |
+
name: NVIDIA H800
|
| 36 |
+
uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
|
| 37 |
+
- architecture: Hopper
|
| 38 |
+
cudaCores: 16896
|
| 39 |
+
memoryTotal: "85520809984"
|
| 40 |
+
name: NVIDIA H800
|
| 41 |
+
uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
|
| 42 |
+
- architecture: Hopper
|
| 43 |
+
cudaCores: 16896
|
| 44 |
+
memoryTotal: "85520809984"
|
| 45 |
+
name: NVIDIA H800
|
| 46 |
+
uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
|
| 47 |
+
- architecture: Hopper
|
| 48 |
+
cudaCores: 16896
|
| 49 |
+
memoryTotal: "85520809984"
|
| 50 |
+
name: NVIDIA H800
|
| 51 |
+
uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
|
| 52 |
+
- architecture: Hopper
|
| 53 |
+
cudaCores: 16896
|
| 54 |
+
memoryTotal: "85520809984"
|
| 55 |
+
name: NVIDIA H800
|
| 56 |
+
uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
|
| 57 |
+
- architecture: Hopper
|
| 58 |
+
cudaCores: 16896
|
| 59 |
+
memoryTotal: "85520809984"
|
| 60 |
+
name: NVIDIA H800
|
| 61 |
+
uuid: GPU-2775489f-18c8-2328-8829-c46238575379
|
| 62 |
+
- architecture: Hopper
|
| 63 |
+
cudaCores: 16896
|
| 64 |
+
memoryTotal: "85520809984"
|
| 65 |
+
name: NVIDIA H800
|
| 66 |
+
uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
|
| 67 |
+
host: b1301b38c75f
|
| 68 |
+
memory:
|
| 69 |
+
total: "2164113813504"
|
| 70 |
+
os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
|
| 71 |
+
program: /data4/runqing/dLLM-RL/train/rl_lladav.py
|
| 72 |
+
python: CPython 3.10.19
|
| 73 |
+
root: /data4/runqing/dLLM-RL
|
| 74 |
+
startedAt: "2026-03-25T06:13:44.186076Z"
|
| 75 |
+
writerId: l1zvo1vvzkyq01b0wblads1a5x42yuiv
|
| 76 |
+
m: []
|
| 77 |
+
python_version: 3.10.19
|
| 78 |
+
t:
|
| 79 |
+
"1":
|
| 80 |
+
- 1
|
| 81 |
+
- 5
|
| 82 |
+
- 11
|
| 83 |
+
- 41
|
| 84 |
+
- 49
|
| 85 |
+
- 53
|
| 86 |
+
- 71
|
| 87 |
+
- 98
|
| 88 |
+
"2":
|
| 89 |
+
- 1
|
| 90 |
+
- 5
|
| 91 |
+
- 11
|
| 92 |
+
- 41
|
| 93 |
+
- 49
|
| 94 |
+
- 51
|
| 95 |
+
- 53
|
| 96 |
+
- 71
|
| 97 |
+
- 98
|
| 98 |
+
"3":
|
| 99 |
+
- 2
|
| 100 |
+
- 13
|
| 101 |
+
- 14
|
| 102 |
+
- 61
|
| 103 |
+
"4": 3.10.19
|
| 104 |
+
"5": 0.25.0
|
| 105 |
+
"6": 4.52.4
|
| 106 |
+
"12": 0.25.0
|
| 107 |
+
"13": linux-x86_64
|
| 108 |
+
config:
|
| 109 |
+
value: configs/rl_lladav_video_reversal.yaml
|
| 110 |
+
dataset.data_type:
|
| 111 |
+
value: video_reverse
|
| 112 |
+
dataset.optimization_data:
|
| 113 |
+
value: rl_data
|
| 114 |
+
dataset.train_dataset:
|
| 115 |
+
value: video_reversal_rl_train
|
| 116 |
+
evaluation.batch_size:
|
| 117 |
+
value: 1
|
| 118 |
+
evaluation.block_size:
|
| 119 |
+
value: 32
|
| 120 |
+
evaluation.data_type:
|
| 121 |
+
value: video_reverse
|
| 122 |
+
evaluation.dynamic_threshold:
|
| 123 |
+
value: 0.9
|
| 124 |
+
evaluation.eval_dataset:
|
| 125 |
+
value: video_reversal_rl_eval
|
| 126 |
+
evaluation.further_horizon:
|
| 127 |
+
value: 128
|
| 128 |
+
evaluation.if_eval:
|
| 129 |
+
value: true
|
| 130 |
+
evaluation.max_gen_length:
|
| 131 |
+
value: 128
|
| 132 |
+
evaluation.num_response_per_task:
|
| 133 |
+
value: 3
|
| 134 |
+
evaluation.remasking_strategy.0:
|
| 135 |
+
value: low_confidence_static
|
| 136 |
+
evaluation.remasking_strategy.1:
|
| 137 |
+
value: low_confidence_dynamic
|
| 138 |
+
evaluation.steps:
|
| 139 |
+
value: 128
|
| 140 |
+
evaluation.target:
|
| 141 |
+
value: confidence
|
| 142 |
+
evaluation.temperature:
|
| 143 |
+
value: 0.1
|
| 144 |
+
evaluation.use_cache:
|
| 145 |
+
value: true
|
| 146 |
+
execute.num_chunk:
|
| 147 |
+
value: 128
|
| 148 |
+
experiment.current_epoch:
|
| 149 |
+
value: 27
|
| 150 |
+
experiment.deepspeed_file:
|
| 151 |
+
value: 1_node_8_gpus_deepspeed_zero2_rl
|
| 152 |
+
experiment.eval_every:
|
| 153 |
+
value: 5
|
| 154 |
+
experiment.function:
|
| 155 |
+
value: train
|
| 156 |
+
experiment.logging_dir:
|
| 157 |
+
value: rl_lladav_video_reversal/logs
|
| 158 |
+
experiment.node_index:
|
| 159 |
+
value: 0
|
| 160 |
+
experiment.num_node:
|
| 161 |
+
value: 1
|
| 162 |
+
experiment.project:
|
| 163 |
+
value: rl_lladav_video_reversal
|
| 164 |
+
experiment.save_every:
|
| 165 |
+
value: 10
|
| 166 |
+
experiment.start_from_scratch:
|
| 167 |
+
value: true
|
| 168 |
+
experiment.total_step:
|
| 169 |
+
value: 30
|
| 170 |
+
lr_scheduler.params.learning_rate:
|
| 171 |
+
value: 1e-06
|
| 172 |
+
lr_scheduler.params.min_lr_scale:
|
| 173 |
+
value: 1
|
| 174 |
+
lr_scheduler.params.warmup_steps:
|
| 175 |
+
value: 0
|
| 176 |
+
lr_scheduler.scheduler:
|
| 177 |
+
value: cosine
|
| 178 |
+
model.image_resolution:
|
| 179 |
+
value: 512
|
| 180 |
+
model.model_base:
|
| 181 |
+
value: lladav
|
| 182 |
+
model.optimized_name:
|
| 183 |
+
value: optimized
|
| 184 |
+
model.pretrained_model:
|
| 185 |
+
value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
|
| 186 |
+
model.vq_model_path:
|
| 187 |
+
value: /data4/runqing/hf/magvitv2
|
| 188 |
+
optimizer.name:
|
| 189 |
+
value: adamw
|
| 190 |
+
optimizer.params.beta1:
|
| 191 |
+
value: 0.9
|
| 192 |
+
optimizer.params.beta2:
|
| 193 |
+
value: 0.999
|
| 194 |
+
optimizer.params.epsilon:
|
| 195 |
+
value: 1e-08
|
| 196 |
+
optimizer.params.learning_rate:
|
| 197 |
+
value: 1e-06
|
| 198 |
+
optimizer.params.scale_lr:
|
| 199 |
+
value: false
|
| 200 |
+
optimizer.params.weight_decay:
|
| 201 |
+
value: 0
|
| 202 |
+
reward.answer_must_in_box:
|
| 203 |
+
value: true
|
| 204 |
+
reward.strict_len_check:
|
| 205 |
+
value: false
|
| 206 |
+
rollout.batch_size:
|
| 207 |
+
value: 1
|
| 208 |
+
rollout.block_size:
|
| 209 |
+
value: 32
|
| 210 |
+
rollout.dynamic_threshold:
|
| 211 |
+
value: 0.9
|
| 212 |
+
rollout.further_horizon:
|
| 213 |
+
value: 128
|
| 214 |
+
rollout.max_gen_length:
|
| 215 |
+
value: 128
|
| 216 |
+
rollout.num_response_per_task:
|
| 217 |
+
value: 8
|
| 218 |
+
rollout.num_task_per_step:
|
| 219 |
+
value: 128
|
| 220 |
+
rollout.remasking_strategy:
|
| 221 |
+
value: low_confidence_dynamic
|
| 222 |
+
rollout.steps:
|
| 223 |
+
value: 128
|
| 224 |
+
rollout.target:
|
| 225 |
+
value: confidence
|
| 226 |
+
rollout.temperature:
|
| 227 |
+
value: 0.8
|
| 228 |
+
rollout.use_cache:
|
| 229 |
+
value: true
|
| 230 |
+
system.HF_HOME:
|
| 231 |
+
value: null
|
| 232 |
+
system.HTTP_PROXY:
|
| 233 |
+
value: null
|
| 234 |
+
system.base_dir:
|
| 235 |
+
value: /data4/runqing/dLLM-RL
|
| 236 |
+
system.env_name:
|
| 237 |
+
value: /data8/runqing/miniconda/envs/dllm-rl
|
| 238 |
+
system.envs_dir:
|
| 239 |
+
value: /data8/runqing/miniconda/envs/dllm-rl
|
| 240 |
+
training.batch_size_lm:
|
| 241 |
+
value: 1
|
| 242 |
+
training.beta:
|
| 243 |
+
value: 0.01
|
| 244 |
+
training.enable_tf32:
|
| 245 |
+
value: true
|
| 246 |
+
training.eps:
|
| 247 |
+
value: 0.2
|
| 248 |
+
training.gradient_accumulation_steps:
|
| 249 |
+
value: 2
|
| 250 |
+
training.lower_p:
|
| 251 |
+
value: 0.35
|
| 252 |
+
training.mask_times_per_sample:
|
| 253 |
+
value: 2
|
| 254 |
+
training.max_gen_length:
|
| 255 |
+
value: 128
|
| 256 |
+
training.max_grad_norm:
|
| 257 |
+
value: 1
|
| 258 |
+
training.max_prompt_len:
|
| 259 |
+
value: 512
|
| 260 |
+
training.method:
|
| 261 |
+
value: random_masking
|
| 262 |
+
training.mixed_precision:
|
| 263 |
+
value: bf16
|
| 264 |
+
training.num_train_epochs:
|
| 265 |
+
value: 1
|
| 266 |
+
training.post_num:
|
| 267 |
+
value: 1
|
| 268 |
+
training.seed:
|
| 269 |
+
value: 10086
|
| 270 |
+
training.shrink:
|
| 271 |
+
value: 8
|
| 272 |
+
training.upper_p:
|
| 273 |
+
value: 0.9
|
| 274 |
+
training.use_kl_estimator_k3:
|
| 275 |
+
value: true
|
| 276 |
+
wandb.entity:
|
| 277 |
+
value: null
|
| 278 |
+
wandb.resume:
|
| 279 |
+
value: auto
|
| 280 |
+
wandb.run_id:
|
| 281 |
+
value: o1lsxhwu
|
wandb/run-20260325_141344-o1lsxhwu/files/diff.patch
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..9f94fa7 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,87 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [
|
| 158 |
+
+ {
|
| 159 |
+
+ "name": "stdout",
|
| 160 |
+
+ "output_type": "stream",
|
| 161 |
+
+ "text": [
|
| 162 |
+
+ "200\n"
|
| 163 |
+
+ ]
|
| 164 |
+
+ },
|
| 165 |
+
+ {
|
| 166 |
+
+ "ename": "",
|
| 167 |
+
+ "evalue": "",
|
| 168 |
+
+ "output_type": "error",
|
| 169 |
+
+ "traceback": [
|
| 170 |
+
+ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
|
| 171 |
+
+ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
|
| 172 |
+
+ "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
|
| 173 |
+
+ "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
|
| 174 |
+
+ ]
|
| 175 |
+
+ }
|
| 176 |
+
+ ],
|
| 177 |
+
+ "source": [
|
| 178 |
+
+ "import json\n",
|
| 179 |
+
+ "import random\n",
|
| 180 |
+
+ "random.seed(2026)\n",
|
| 181 |
+
+ "\n",
|
| 182 |
+
+ "\n",
|
| 183 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
|
| 184 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
|
| 185 |
+
+ "save_data_list = []\n",
|
| 186 |
+
+ "save_num = 0\n",
|
| 187 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 188 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 189 |
+
+ " raw_data = json.load(fin)\n",
|
| 190 |
+
+ "\n",
|
| 191 |
+
+ " # random.shuffle(raw_data)\n",
|
| 192 |
+
+ " for did, data in enumerate(raw_data):\n",
|
| 193 |
+
+ " # if did > 1999:\n",
|
| 194 |
+
+ " if save_num < 200:\n",
|
| 195 |
+
+ " save_data_list.append(data)\n",
|
| 196 |
+
+ " save_num += 1\n",
|
| 197 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 198 |
+
+ " print(len(save_data_list))\n"
|
| 199 |
+
+ ]
|
| 200 |
+
}
|
| 201 |
+
],
|
| 202 |
+
"metadata": {
|
| 203 |
+
@@ -85,7 +166,7 @@
|
| 204 |
+
"name": "python",
|
| 205 |
+
"nbconvert_exporter": "python",
|
| 206 |
+
"pygments_lexer": "ipython3",
|
| 207 |
+
- "version": "3.10.18"
|
| 208 |
+
+ "version": "3.10.19"
|
| 209 |
+
}
|
| 210 |
+
},
|
| 211 |
+
"nbformat": 4,
|
| 212 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 213 |
+
index 8f2c0f1..79d2e5b 100644
|
| 214 |
+
--- a/rl_v.py
|
| 215 |
+
+++ b/rl_v.py
|
| 216 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 217 |
+
def env_prefix(env) -> str:
|
| 218 |
+
return (
|
| 219 |
+
"source ~/.bashrc && "
|
| 220 |
+
- f"source activate {env} && "
|
| 221 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 222 |
+
+ "conda init && "
|
| 223 |
+
+ f"conda activate {env} && "
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
config = get_config()
|
| 227 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 228 |
+
index 4068fc8..d284a23 100644
|
| 229 |
+
--- a/sample/llava/conversation.py
|
| 230 |
+
+++ b/sample/llava/conversation.py
|
| 231 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 232 |
+
if self.version == "llama_v3":
|
| 233 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 234 |
+
elif self.version == "llava_llada":
|
| 235 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 236 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 237 |
+
else:
|
| 238 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 239 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 240 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 241 |
+
index 3db5302..96e33b5 100644
|
| 242 |
+
--- a/sample/llava/model/builder.py
|
| 243 |
+
+++ b/sample/llava/model/builder.py
|
| 244 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 245 |
+
elif "llada" in model_name.lower():
|
| 246 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 247 |
+
|
| 248 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 249 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 250 |
+
+
|
| 251 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 252 |
+
if customized_config is None:
|
| 253 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 254 |
+
else:
|
| 255 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 256 |
+
index 4068fc8..d284a23 100644
|
| 257 |
+
--- a/train/llava/conversation.py
|
| 258 |
+
+++ b/train/llava/conversation.py
|
| 259 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 260 |
+
if self.version == "llama_v3":
|
| 261 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 262 |
+
elif self.version == "llava_llada":
|
| 263 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 264 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 265 |
+
else:
|
| 266 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 267 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 268 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 269 |
+
index 3db5302..aafab95 100644
|
| 270 |
+
--- a/train/llava/model/builder.py
|
| 271 |
+
+++ b/train/llava/model/builder.py
|
| 272 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 273 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 274 |
+
elif "llada" in model_name.lower():
|
| 275 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 276 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 277 |
+
|
| 278 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 279 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 280 |
+
if customized_config is None:
|
| 281 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 282 |
+
else:
|
| 283 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 284 |
+
index 3da28dc..8b17da3 100644
|
| 285 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 286 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 287 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 288 |
+
dpo_forward: Optional[bool] = None,
|
| 289 |
+
cache_position=None,
|
| 290 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 291 |
+
-
|
| 292 |
+
+ conversation_ids = None
|
| 293 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 294 |
+
# donate multi-dialogue
|
| 295 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
| 296 |
+
diff --git a/train/rl_llada.py b/train/rl_llada.py
|
| 297 |
+
index 916ce23..00693be 100644
|
| 298 |
+
--- a/train/rl_llada.py
|
| 299 |
+
+++ b/train/rl_llada.py
|
| 300 |
+
@@ -96,7 +96,7 @@ def main():
|
| 301 |
+
accelerator = Accelerator(
|
| 302 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 303 |
+
mixed_precision=config.training.mixed_precision,
|
| 304 |
+
- log_with=None,
|
| 305 |
+
+ log_with='wandb',
|
| 306 |
+
project_dir=config.experiment.logging_dir,
|
| 307 |
+
split_batches=True,
|
| 308 |
+
)
|
wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
|
| 2 |
+
index 3e07fbd..337824b 100644
|
| 3 |
+
--- a/configs/lladav_eval.yaml
|
| 4 |
+
+++ b/configs/lladav_eval.yaml
|
| 5 |
+
@@ -7,17 +7,17 @@ system:
|
| 6 |
+
HTTP_PROXY: null
|
| 7 |
+
HF_HOME: null
|
| 8 |
+
env_name: ""
|
| 9 |
+
- envs_dir: "/abs/path/of/envs"
|
| 10 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 11 |
+
+ envs_dir: null
|
| 12 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 13 |
+
|
| 14 |
+
-model: "/abs/path/of/model"
|
| 15 |
+
+model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 16 |
+
model_base: "lladav"
|
| 17 |
+
answer_must_in_box: True
|
| 18 |
+
|
| 19 |
+
dataset:
|
| 20 |
+
eval_dataset: "SEEDBench_IMG_32"
|
| 21 |
+
data_type: "mmu"
|
| 22 |
+
- image_root: "/abs/path/of/data/images"
|
| 23 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 24 |
+
|
| 25 |
+
execute:
|
| 26 |
+
num_chunk: 128
|
| 27 |
+
@@ -35,4 +35,3 @@ rollout:
|
| 28 |
+
further_horizon: 128
|
| 29 |
+
output_unmasking_history: True
|
| 30 |
+
use_cache: True
|
| 31 |
+
-
|
| 32 |
+
diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
|
| 33 |
+
index 7e38050..a07f620 100644
|
| 34 |
+
--- a/configs/rl_lladav.yaml
|
| 35 |
+
+++ b/configs/rl_lladav.yaml
|
| 36 |
+
@@ -5,9 +5,9 @@ wandb:
|
| 37 |
+
system:
|
| 38 |
+
HTTP_PROXY: null
|
| 39 |
+
HF_HOME: null
|
| 40 |
+
- env_name: "env_name"
|
| 41 |
+
- envs_dir: "/abs/path/of/envs"
|
| 42 |
+
- base_dir: "/abs/path/of/dLLM-RL"
|
| 43 |
+
+ env_name: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 44 |
+
+ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
|
| 45 |
+
+ base_dir: "/data4/runqing/dLLM-RL"
|
| 46 |
+
|
| 47 |
+
experiment:
|
| 48 |
+
project: "rl_lladav"
|
| 49 |
+
@@ -22,17 +22,17 @@ experiment:
|
| 50 |
+
node_index: 0
|
| 51 |
+
|
| 52 |
+
model:
|
| 53 |
+
- pretrained_model: "/abs/path/of/sft-ckpt"
|
| 54 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 55 |
+
optimized_name: "optimized"
|
| 56 |
+
model_base: "lladav"
|
| 57 |
+
- vq_model_path: "/abs/path/of/magvitv2"
|
| 58 |
+
+ vq_model_path: "/data4/runqing/hf/magvitv2"
|
| 59 |
+
image_resolution: 512
|
| 60 |
+
|
| 61 |
+
dataset:
|
| 62 |
+
train_dataset: "SEEDBench_IMG_32"
|
| 63 |
+
optimization_data: "rl_data"
|
| 64 |
+
data_type: "mmu"
|
| 65 |
+
- image_root: "/abs/path/of/data/images"
|
| 66 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 67 |
+
|
| 68 |
+
rollout:
|
| 69 |
+
num_task_per_step: 128
|
| 70 |
+
diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
|
| 71 |
+
index 6557210..001a6e2 100644
|
| 72 |
+
--- a/configs/sft_lladav.yaml
|
| 73 |
+
+++ b/configs/sft_lladav.yaml
|
| 74 |
+
@@ -7,13 +7,13 @@ experiment:
|
| 75 |
+
num_node: 1
|
| 76 |
+
|
| 77 |
+
model:
|
| 78 |
+
- pretrained_model: "/abs/path/of/model"
|
| 79 |
+
+ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
|
| 80 |
+
optimized_name: "optimized"
|
| 81 |
+
|
| 82 |
+
dataset:
|
| 83 |
+
- optimization_data: "sft_test" # ./data/sft_test.json
|
| 84 |
+
+ optimization_data: "sft_test_v" # ./data/sft_test.json
|
| 85 |
+
data_type: "mmu"
|
| 86 |
+
- image_root: "/base_dir/data/images"
|
| 87 |
+
+ image_root: "/data4/runqing/dLLM-RL/data/images"
|
| 88 |
+
|
| 89 |
+
training:
|
| 90 |
+
gradient_checkpointing_enable: False
|
| 91 |
+
diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
|
| 92 |
+
index e0b183f..9f94fa7 100644
|
| 93 |
+
--- a/data/preprocess_sft.ipynb
|
| 94 |
+
+++ b/data/preprocess_sft.ipynb
|
| 95 |
+
@@ -30,15 +30,15 @@
|
| 96 |
+
"#eos_token = \"<|im_end|>\"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# llada and mmada:\n",
|
| 99 |
+
- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 100 |
+
- "#eos_token = \"<|eot_id|>\"\n",
|
| 101 |
+
+ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
|
| 102 |
+
+ "eos_token = \"<|eot_id|>\"\n",
|
| 103 |
+
"\n",
|
| 104 |
+
"# trado and sdar:\n",
|
| 105 |
+
"# non-cot prompt, we used this as demon example to compare sft methods\n",
|
| 106 |
+
- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 107 |
+
+ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 108 |
+
"# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
|
| 109 |
+
"#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
|
| 110 |
+
- "eos_token = \"<|im_end|>\"\n",
|
| 111 |
+
+ "# eos_token = \"<|im_end|>\"\n",
|
| 112 |
+
"\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"def get_prompt(data_i):\n",
|
| 115 |
+
@@ -67,6 +67,87 @@
|
| 116 |
+
"with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
|
| 117 |
+
" json.dump(processed_data, f, indent=2, ensure_ascii=False)"
|
| 118 |
+
]
|
| 119 |
+
+ },
|
| 120 |
+
+ {
|
| 121 |
+
+ "cell_type": "code",
|
| 122 |
+
+ "execution_count": 2,
|
| 123 |
+
+ "id": "32bbd872",
|
| 124 |
+
+ "metadata": {},
|
| 125 |
+
+ "outputs": [
|
| 126 |
+
+ {
|
| 127 |
+
+ "name": "stdout",
|
| 128 |
+
+ "output_type": "stream",
|
| 129 |
+
+ "text": [
|
| 130 |
+
+ "100\n"
|
| 131 |
+
+ ]
|
| 132 |
+
+ }
|
| 133 |
+
+ ],
|
| 134 |
+
+ "source": [
|
| 135 |
+
+ "import json\n",
|
| 136 |
+
+ "\n",
|
| 137 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
|
| 138 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
|
| 139 |
+
+ "save_data_list = []\n",
|
| 140 |
+
+ "save_num = 0\n",
|
| 141 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 142 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 143 |
+
+ " raw_data = json.load(fin)\n",
|
| 144 |
+
+ " for data in raw_data:\n",
|
| 145 |
+
+ " if save_num < 100:\n",
|
| 146 |
+
+ " save_data_list.append(data)\n",
|
| 147 |
+
+ " save_num += 1\n",
|
| 148 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 149 |
+
+ " print(len(save_data_list))\n"
|
| 150 |
+
+ ]
|
| 151 |
+
+ },
|
| 152 |
+
+ {
|
| 153 |
+
+ "cell_type": "code",
|
| 154 |
+
+ "execution_count": null,
|
| 155 |
+
+ "id": "fdfde30f",
|
| 156 |
+
+ "metadata": {},
|
| 157 |
+
+ "outputs": [
|
| 158 |
+
+ {
|
| 159 |
+
+ "name": "stdout",
|
| 160 |
+
+ "output_type": "stream",
|
| 161 |
+
+ "text": [
|
| 162 |
+
+ "200\n"
|
| 163 |
+
+ ]
|
| 164 |
+
+ },
|
| 165 |
+
+ {
|
| 166 |
+
+ "ename": "",
|
| 167 |
+
+ "evalue": "",
|
| 168 |
+
+ "output_type": "error",
|
| 169 |
+
+ "traceback": [
|
| 170 |
+
+ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
|
| 171 |
+
+ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
|
| 172 |
+
+ "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
|
| 173 |
+
+ "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
|
| 174 |
+
+ ]
|
| 175 |
+
+ }
|
| 176 |
+
+ ],
|
| 177 |
+
+ "source": [
|
| 178 |
+
+ "import json\n",
|
| 179 |
+
+ "import random\n",
|
| 180 |
+
+ "random.seed(2026)\n",
|
| 181 |
+
+ "\n",
|
| 182 |
+
+ "\n",
|
| 183 |
+
+ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
|
| 184 |
+
+ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
|
| 185 |
+
+ "save_data_list = []\n",
|
| 186 |
+
+ "save_num = 0\n",
|
| 187 |
+
+ "with open(data_path, 'r') as fin, \\\n",
|
| 188 |
+
+ " open(save_path, 'w') as fout:\n",
|
| 189 |
+
+ " raw_data = json.load(fin)\n",
|
| 190 |
+
+ "\n",
|
| 191 |
+
+ " # random.shuffle(raw_data)\n",
|
| 192 |
+
+ " for did, data in enumerate(raw_data):\n",
|
| 193 |
+
+ " # if did > 1999:\n",
|
| 194 |
+
+ " if save_num < 200:\n",
|
| 195 |
+
+ " save_data_list.append(data)\n",
|
| 196 |
+
+ " save_num += 1\n",
|
| 197 |
+
+ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
|
| 198 |
+
+ " print(len(save_data_list))\n"
|
| 199 |
+
+ ]
|
| 200 |
+
}
|
| 201 |
+
],
|
| 202 |
+
"metadata": {
|
| 203 |
+
@@ -85,7 +166,7 @@
|
| 204 |
+
"name": "python",
|
| 205 |
+
"nbconvert_exporter": "python",
|
| 206 |
+
"pygments_lexer": "ipython3",
|
| 207 |
+
- "version": "3.10.18"
|
| 208 |
+
+ "version": "3.10.19"
|
| 209 |
+
}
|
| 210 |
+
},
|
| 211 |
+
"nbformat": 4,
|
| 212 |
+
diff --git a/rl_v.py b/rl_v.py
|
| 213 |
+
index 8f2c0f1..79d2e5b 100644
|
| 214 |
+
--- a/rl_v.py
|
| 215 |
+
+++ b/rl_v.py
|
| 216 |
+
@@ -15,7 +15,9 @@ if __name__ == "__main__":
|
| 217 |
+
def env_prefix(env) -> str:
|
| 218 |
+
return (
|
| 219 |
+
"source ~/.bashrc && "
|
| 220 |
+
- f"source activate {env} && "
|
| 221 |
+
+ "source /data8/runqing/miniconda/bin/activate && "
|
| 222 |
+
+ "conda init && "
|
| 223 |
+
+ f"conda activate {env} && "
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
config = get_config()
|
| 227 |
+
diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
|
| 228 |
+
index 4068fc8..d284a23 100644
|
| 229 |
+
--- a/sample/llava/conversation.py
|
| 230 |
+
+++ b/sample/llava/conversation.py
|
| 231 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 232 |
+
if self.version == "llama_v3":
|
| 233 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 234 |
+
elif self.version == "llava_llada":
|
| 235 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 236 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 237 |
+
else:
|
| 238 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 239 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 240 |
+
diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
|
| 241 |
+
index 3db5302..96e33b5 100644
|
| 242 |
+
--- a/sample/llava/model/builder.py
|
| 243 |
+
+++ b/sample/llava/model/builder.py
|
| 244 |
+
@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 245 |
+
elif "llada" in model_name.lower():
|
| 246 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 247 |
+
|
| 248 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 249 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 250 |
+
+
|
| 251 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 252 |
+
if customized_config is None:
|
| 253 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 254 |
+
else:
|
| 255 |
+
diff --git a/train/llava/conversation.py b/train/llava/conversation.py
|
| 256 |
+
index 4068fc8..d284a23 100644
|
| 257 |
+
--- a/train/llava/conversation.py
|
| 258 |
+
+++ b/train/llava/conversation.py
|
| 259 |
+
@@ -99,7 +99,7 @@ class Conversation:
|
| 260 |
+
if self.version == "llama_v3":
|
| 261 |
+
self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
|
| 262 |
+
elif self.version == "llava_llada":
|
| 263 |
+
- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
|
| 264 |
+
+ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
|
| 265 |
+
else:
|
| 266 |
+
raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
|
| 267 |
+
chat_template_messages = [{"role": "system", "content": self.system}]
|
| 268 |
+
diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
|
| 269 |
+
index 3db5302..aafab95 100644
|
| 270 |
+
--- a/train/llava/model/builder.py
|
| 271 |
+
+++ b/train/llava/model/builder.py
|
| 272 |
+
@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
|
| 273 |
+
model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
|
| 274 |
+
elif "llada" in model_name.lower():
|
| 275 |
+
from llava.model.language_model.llava_llada import LlavaLLaDAConfig
|
| 276 |
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
|
| 277 |
+
|
| 278 |
+
- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 279 |
+
+ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
|
| 280 |
+
if customized_config is None:
|
| 281 |
+
llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
|
| 282 |
+
else:
|
| 283 |
+
diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
|
| 284 |
+
index 3da28dc..8b17da3 100644
|
| 285 |
+
--- a/train/llava/model/language_model/llava_llada.py
|
| 286 |
+
+++ b/train/llava/model/language_model/llava_llada.py
|
| 287 |
+
@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
|
| 288 |
+
dpo_forward: Optional[bool] = None,
|
| 289 |
+
cache_position=None,
|
| 290 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 291 |
+
-
|
| 292 |
+
+ conversation_ids = None
|
| 293 |
+
if inputs_embeds is None and attention_mask is not None:
|
| 294 |
+
# donate multi-dialogue
|
| 295 |
+
(input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
|
| 296 |
+
diff --git a/train/rl_llada.py b/train/rl_llada.py
|
| 297 |
+
index 916ce23..00693be 100644
|
| 298 |
+
--- a/train/rl_llada.py
|
| 299 |
+
+++ b/train/rl_llada.py
|
| 300 |
+
@@ -96,7 +96,7 @@ def main():
|
| 301 |
+
accelerator = Accelerator(
|
| 302 |
+
gradient_accumulation_steps=config.training.gradient_accumulation_steps,
|
| 303 |
+
mixed_precision=config.training.mixed_precision,
|
| 304 |
+
- log_with=None,
|
| 305 |
+
+ log_with='wandb',
|
| 306 |
+
project_dir=config.experiment.logging_dir,
|
| 307 |
+
split_batches=True,
|
| 308 |
+
)
|
wandb/run-20260325_141344-o1lsxhwu/files/output.log
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
03/25/2026 14:13:47 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml
|
| 2 |
+
03/25/2026 14:13:47 - INFO - __main__ - Loading LLaDA-V model and tokenizer
|
| 3 |
+
Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal/ckpt/optimized
|
| 4 |
+
Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
|
| 5 |
+
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.72s/it]
|
| 6 |
+
Rank 0: Model Class: LlavaLLaDAModelLM
|
| 7 |
+
03/25/2026 14:14:14 - INFO - __main__ - Preprocessing Data
|
| 8 |
+
03/25/2026 14:14:14 - INFO - __main__ - Preparing model, optimizer and dataloaders
|
| 9 |
+
/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
|
| 10 |
+
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
|
| 11 |
+
warnings.warn(
|
| 12 |
+
Before initializing optimizer states
|
| 13 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 14 |
+
CPU Virtual Memory: used = 181.57 GB, percent = 9.0%
|
| 15 |
+
After initializing optimizer states
|
| 16 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 17 |
+
CPU Virtual Memory: used = 194.7 GB, percent = 9.7%
|
| 18 |
+
After initializing ZeRO optimizer
|
| 19 |
+
MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
|
| 20 |
+
CPU Virtual Memory: used = 194.31 GB, percent = 9.6%
|
| 21 |
+
03/25/2026 14:15:29 - INFO - __main__ - ***** Running inference *****
|
| 22 |
+
03/25/2026 14:15:29 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants...
|
| 23 |
+
03/25/2026 14:19:13 - INFO - __main__ - ***** Running training *****
|
| 24 |
+
03/25/2026 14:19:13 - INFO - __main__ - Num variants = 560
|
| 25 |
+
03/25/2026 14:19:13 - INFO - __main__ - Num training steps = 36
|
| 26 |
+
03/25/2026 14:19:13 - INFO - __main__ - Instantaneous batch size per device = 1
|
| 27 |
+
03/25/2026 14:19:13 - INFO - __main__ - Total train batch size (parallel * accum) = 16
|
| 28 |
+
03/25/2026 14:19:13 - INFO - __main__ - Gradient Accumulation steps = 2
|
| 29 |
+
start rl training...
|
| 30 |
+
Epoch 1/1: 100%|██████████| 70/70 [18:59<00:00, 16.28s/it]
|
| 31 |
+
03/25/2026 14:38:51 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal/ckpt/optimized
|
wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==4.52.4
|
| 2 |
+
nvidia-nvtx-cu12==12.4.127
|
| 3 |
+
deepspeed==0.18.6
|
| 4 |
+
typing-inspection==0.4.2
|
| 5 |
+
scipy==1.15.3
|
| 6 |
+
MarkupSafe==3.0.3
|
| 7 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 8 |
+
yarl==1.22.0
|
| 9 |
+
tornado==6.5.4
|
| 10 |
+
nvidia-curand-cu12==10.3.5.147
|
| 11 |
+
nest-asyncio==1.6.0
|
| 12 |
+
safetensors==0.7.0
|
| 13 |
+
async-timeout==5.0.1
|
| 14 |
+
typing_extensions==4.15.0
|
| 15 |
+
asttokens==3.0.1
|
| 16 |
+
cuda-python==13.1.1
|
| 17 |
+
executing==2.2.1
|
| 18 |
+
nvidia-cudnn-frontend==1.18.0
|
| 19 |
+
comm==0.2.3
|
| 20 |
+
tokenizers==0.21.4
|
| 21 |
+
six==1.17.0
|
| 22 |
+
Pygments==2.19.2
|
| 23 |
+
nvidia-cusparselt-cu12==0.6.2
|
| 24 |
+
apache-tvm-ffi==0.1.8.post2
|
| 25 |
+
flash_attn==2.7.4.post1
|
| 26 |
+
tqdm==4.67.3
|
| 27 |
+
tensorboard-data-server==0.7.2
|
| 28 |
+
annotated-types==0.7.0
|
| 29 |
+
math-verify==0.9.0
|
| 30 |
+
idna==3.11
|
| 31 |
+
regex==2026.1.15
|
| 32 |
+
nvidia-cutlass-dsl-libs-base==4.4.0
|
| 33 |
+
flashinfer-python==0.6.3
|
| 34 |
+
click==8.3.1
|
| 35 |
+
pytz==2025.2
|
| 36 |
+
wheel==0.46.3
|
| 37 |
+
ipykernel==7.2.0
|
| 38 |
+
stack-data==0.6.3
|
| 39 |
+
absl-py==2.4.0
|
| 40 |
+
wandb==0.25.0
|
| 41 |
+
pandas==2.3.3
|
| 42 |
+
termcolor==3.3.0
|
| 43 |
+
sympy==1.13.1
|
| 44 |
+
joblib==1.5.3
|
| 45 |
+
prompt_toolkit==3.0.52
|
| 46 |
+
attrs==25.4.0
|
| 47 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 48 |
+
xxhash==3.6.0
|
| 49 |
+
aiohappyeyeballs==2.6.1
|
| 50 |
+
python-dateutil==2.9.0.post0
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nvidia-ml-py==13.590.48
|
| 53 |
+
multiprocess==0.70.18
|
| 54 |
+
decorator==5.2.1
|
| 55 |
+
cuda-pathfinder==1.3.4
|
| 56 |
+
multidict==6.7.1
|
| 57 |
+
pure_eval==0.2.3
|
| 58 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 59 |
+
omegaconf==2.3.0
|
| 60 |
+
PyYAML==6.0.3
|
| 61 |
+
debugpy==1.8.20
|
| 62 |
+
psutil==7.2.2
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 65 |
+
triton==3.2.0
|
| 66 |
+
antlr4-python3-runtime==4.9.3
|
| 67 |
+
peft==0.18.1
|
| 68 |
+
aiosignal==1.4.0
|
| 69 |
+
accelerate==1.12.0
|
| 70 |
+
liger_kernel==0.7.0
|
| 71 |
+
anyio==4.12.1
|
| 72 |
+
decord==0.6.0
|
| 73 |
+
h11==0.16.0
|
| 74 |
+
urllib3==2.6.3
|
| 75 |
+
wcwidth==0.6.0
|
| 76 |
+
networkx==3.4.2
|
| 77 |
+
huggingface_hub==0.36.2
|
| 78 |
+
tzdata==2025.3
|
| 79 |
+
xformers==0.0.29.post3
|
| 80 |
+
tabulate==0.9.0
|
| 81 |
+
certifi==2026.1.4
|
| 82 |
+
tensorboard==2.20.0
|
| 83 |
+
matplotlib-inline==0.2.1
|
| 84 |
+
pip==26.0.1
|
| 85 |
+
smmap==5.0.2
|
| 86 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 87 |
+
Markdown==3.10.2
|
| 88 |
+
nvidia-nccl-cu12==2.21.5
|
| 89 |
+
aiohttp==3.13.3
|
| 90 |
+
scikit-learn==1.7.2
|
| 91 |
+
nvidia-cutlass-dsl==4.4.0
|
| 92 |
+
setuptools==82.0.0
|
| 93 |
+
jupyter_core==5.9.1
|
| 94 |
+
asyncio==4.0.0
|
| 95 |
+
einops==0.8.2
|
| 96 |
+
pydantic==2.12.5
|
| 97 |
+
av==16.1.0
|
| 98 |
+
protobuf==6.33.5
|
| 99 |
+
evaluate==0.4.6
|
| 100 |
+
Jinja2==3.1.6
|
| 101 |
+
exceptiongroup==1.3.1
|
| 102 |
+
ipython==8.38.0
|
| 103 |
+
sentry-sdk==2.53.0
|
| 104 |
+
httpx==0.28.1
|
| 105 |
+
msgpack==1.1.2
|
| 106 |
+
platformdirs==4.9.2
|
| 107 |
+
hjson==3.1.0
|
| 108 |
+
torchvision==0.21.0
|
| 109 |
+
parso==0.8.6
|
| 110 |
+
cuda-bindings==13.1.1
|
| 111 |
+
pexpect==4.9.0
|
| 112 |
+
threadpoolctl==3.6.0
|
| 113 |
+
pyarrow==23.0.1
|
| 114 |
+
jedi==0.19.2
|
| 115 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 116 |
+
Werkzeug==3.1.5
|
| 117 |
+
fsspec==2025.10.0
|
| 118 |
+
pydantic_core==2.41.5
|
| 119 |
+
GitPython==3.1.46
|
| 120 |
+
propcache==0.4.1
|
| 121 |
+
mpmath==1.3.0
|
| 122 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 123 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 124 |
+
hf-xet==1.2.0
|
| 125 |
+
charset-normalizer==3.4.4
|
| 126 |
+
dill==0.4.0
|
| 127 |
+
datasets==4.5.0
|
| 128 |
+
py-cpuinfo==9.0.0
|
| 129 |
+
sentencepiece==0.2.1
|
| 130 |
+
latex2sympy2_extended==1.11.0
|
| 131 |
+
jupyter_client==8.8.0
|
| 132 |
+
pyzmq==27.1.0
|
| 133 |
+
numpy==2.2.6
|
| 134 |
+
gitdb==4.0.12
|
| 135 |
+
grpcio==1.78.0
|
| 136 |
+
packaging==26.0
|
| 137 |
+
requests==2.32.5
|
| 138 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 139 |
+
httpcore==1.0.9
|
| 140 |
+
traitlets==5.14.3
|
| 141 |
+
filelock==3.24.2
|
| 142 |
+
ptyprocess==0.7.0
|
| 143 |
+
pillow==12.1.1
|
| 144 |
+
torch==2.6.0
|
| 145 |
+
more-itertools==10.8.0
|
| 146 |
+
wheel==0.46.3
|
| 147 |
+
jaraco.text==4.0.0
|
| 148 |
+
zipp==3.23.0
|
| 149 |
+
autocommand==2.2.2
|
| 150 |
+
jaraco.context==6.1.0
|
| 151 |
+
jaraco.functools==4.4.0
|
| 152 |
+
tomli==2.4.0
|
| 153 |
+
backports.tarfile==1.2.0
|
| 154 |
+
importlib_metadata==8.7.1
|
| 155 |
+
platformdirs==4.4.0
|
| 156 |
+
packaging==26.0
|
wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-03-25T06:13:44.186076Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"config=configs/rl_lladav_video_reversal.yaml",
|
| 7 |
+
"experiment.current_epoch=27"
|
| 8 |
+
],
|
| 9 |
+
"program": "/data4/runqing/dLLM-RL/train/rl_lladav.py",
|
| 10 |
+
"codePath": "train/rl_lladav.py",
|
| 11 |
+
"codePathLocal": "train/rl_lladav.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
|
| 14 |
+
"commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
|
| 15 |
+
},
|
| 16 |
+
"email": "shixi_runqing@staff.weibo.com",
|
| 17 |
+
"root": "/data4/runqing/dLLM-RL",
|
| 18 |
+
"host": "b1301b38c75f",
|
| 19 |
+
"executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H800",
|
| 23 |
+
"gpu_count": 8,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "806578724864",
|
| 27 |
+
"used": "572884140032"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "2164113813504"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H800",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H800",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H800",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H800",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H800",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H800",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"name": "NVIDIA H800",
|
| 78 |
+
"memoryTotal": "85520809984",
|
| 79 |
+
"cudaCores": 16896,
|
| 80 |
+
"architecture": "Hopper",
|
| 81 |
+
"uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "NVIDIA H800",
|
| 85 |
+
"memoryTotal": "85520809984",
|
| 86 |
+
"cudaCores": 16896,
|
| 87 |
+
"architecture": "Hopper",
|
| 88 |
+
"uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
|
| 89 |
+
}
|
| 90 |
+
],
|
| 91 |
+
"cudaVersion": "12.4",
|
| 92 |
+
"writerId": "l1zvo1vvzkyq01b0wblads1a5x42yuiv"
|
| 93 |
+
}
|
wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_timestamp":1.774420691877827e+09,"_wandb":{"runtime":1504},"_runtime":1504.468829676,"_step":35,"train/loss":-1.004162073135376,"train/learning_rate":5.868240888334637e-07}
|