GuoruiSong commited on
Commit
bb0ed79
·
verified ·
1 Parent(s): 90c7f8f

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py +750 -0
  2. wandb/run-20260319_223122-agxw5i56/files/diff.patch +267 -0
  3. wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +267 -0
  4. wandb/run-20260319_223122-agxw5i56/files/output.log +26 -0
  5. wandb/run-20260319_223122-agxw5i56/files/requirements.txt +156 -0
  6. wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json +92 -0
  7. wandb/run-20260319_223122-agxw5i56/logs/debug-core.log +8 -0
  8. wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log +18 -0
  9. wandb/run-20260319_223122-agxw5i56/logs/debug.log +20 -0
  10. wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py +757 -0
  11. wandb/run-20260322_031609-y30zeekk/files/diff.patch +270 -0
  12. wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +270 -0
  13. wandb/run-20260322_031609-y30zeekk/files/output.log +26 -0
  14. wandb/run-20260322_031609-y30zeekk/files/requirements.txt +156 -0
  15. wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json +92 -0
  16. wandb/run-20260322_031609-y30zeekk/logs/debug-core.log +7 -0
  17. wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log +18 -0
  18. wandb/run-20260322_031609-y30zeekk/logs/debug.log +20 -0
  19. wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb +0 -0
  20. wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py +806 -0
  21. wandb/run-20260323_031014-fmwqi7oc/files/config.yaml +279 -0
  22. wandb/run-20260323_031014-fmwqi7oc/files/diff.patch +270 -0
  23. wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +270 -0
  24. wandb/run-20260323_031014-fmwqi7oc/files/output.log +82 -0
  25. wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt +156 -0
  26. wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json +93 -0
  27. wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json +1 -0
  28. wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log +15 -0
  29. wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log +29 -0
  30. wandb/run-20260323_031014-fmwqi7oc/logs/debug.log +22 -0
  31. wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb +0 -0
  32. wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py +815 -0
  33. wandb/run-20260324_164425-eb44q2tb/files/config.yaml +279 -0
  34. wandb/run-20260324_164425-eb44q2tb/files/diff.patch +308 -0
  35. wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +308 -0
  36. wandb/run-20260324_164425-eb44q2tb/files/output.log +42 -0
  37. wandb/run-20260324_164425-eb44q2tb/files/requirements.txt +156 -0
  38. wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json +93 -0
  39. wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json +1 -0
  40. wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log +15 -0
  41. wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log +29 -0
  42. wandb/run-20260324_164425-eb44q2tb/logs/debug.log +22 -0
  43. wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py +825 -0
  44. wandb/run-20260325_141344-o1lsxhwu/files/config.yaml +281 -0
  45. wandb/run-20260325_141344-o1lsxhwu/files/diff.patch +308 -0
  46. wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch +308 -0
  47. wandb/run-20260325_141344-o1lsxhwu/files/output.log +31 -0
  48. wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt +156 -0
  49. wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json +93 -0
  50. wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json +1 -0
wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py ADDED
@@ -0,0 +1,750 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
4
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
5
+ from datetime import timedelta
6
+ import json
7
+ import logging
8
+ import math
9
+ import shutil
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Dict, List, Any
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from torch.optim import AdamW
18
+ from torch.utils.data import Dataset, DataLoader
19
+
20
+ from omegaconf import OmegaConf
21
+ import wandb
22
+
23
+ from accelerate import Accelerator, InitProcessGroupKwargs
24
+ from accelerate.logging import get_logger
25
+ from accelerate.utils import set_seed
26
+
27
+ from train.utils import get_config, flatten_omega_conf, AverageMeter
28
+ from models.lr_schedulers import get_scheduler
29
+ from models.logging import set_verbosity_info, set_verbosity_error
30
+
31
+ # LLaDA-V / LLaVA
32
+ from transformers.utils import logging as hf_logging
33
+ hf_logging.set_verbosity_error()
34
+
35
+ from llava.model.builder import load_pretrained_model
36
+ from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord
37
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
38
+ from llava.conversation import conv_templates
39
+ # from accelerate import DistributedDataParallelKwargs #添加
40
+
41
+ logger = get_logger(__name__, log_level="INFO")
42
+
43
+
44
+ def collapse_k_unique(lst, k: int):
45
+ if k <= 0:
46
+ raise ValueError("k must be > 0")
47
+ uniq = sorted(set(lst))
48
+ mapping = {}
49
+ n = len(uniq)
50
+ for idx, val in enumerate(uniq):
51
+ group = idx // k
52
+ end_idx = min((group + 1) * k - 1, n - 1)
53
+ rep = uniq[end_idx]
54
+ mapping[val] = rep
55
+ return [mapping[x] for x in lst]
56
+
57
+
58
+ def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
59
+ import copy
60
+ conv = copy.deepcopy(conv_templates["llava_llada"])
61
+ conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
62
+ conv.append_message(conv.roles[1], response_text)
63
+ return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
64
+
65
+
66
+ def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
67
+ conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
68
+ ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
69
+ return ids.squeeze(0) # (L,)
70
+
71
+
72
+ def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
73
+ """
74
+ 统一处理 Image 和 Video
75
+ 返回 (media_tensor_list, sizes, video_stats)
76
+ """
77
+ from PIL import Image
78
+ from types import SimpleNamespace
79
+
80
+ if isinstance(model_cfg, dict):
81
+ model_cfg = SimpleNamespace(**model_cfg)
82
+
83
+ if media_type == "image":
84
+ image = Image.open(abs_path).convert("RGB")
85
+ if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
86
+ setattr(model_cfg, "image_aspect_ratio", "pad")
87
+ if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
88
+ setattr(model_cfg, "mm_patch_merge_type", "flat")
89
+
90
+ image_tensor_list = process_images([image], image_processor, model_cfg)
91
+ image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
92
+ image_sizes = [image.size]
93
+ return image_tensor_list, image_sizes, None
94
+
95
+ elif media_type == "video":
96
+ if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
97
+ if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 0)
98
+ if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", False)
99
+
100
+ # 使用官方提供的 decord 函数处理视频
101
+ video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
102
+
103
+ # 处理成 tensor
104
+ image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
105
+ image_tensor = image_tensor.to(dtype=dtype, device=device)
106
+
107
+ # 提取 size
108
+ if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
109
+ v_size = (video_frames.shape[2], video_frames.shape[1])
110
+ else:
111
+ v_size = (0, 0)
112
+
113
+ return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
114
+
115
+
116
+ def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
117
+ out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
118
+ out[-x.numel():] = x
119
+ return out
120
+
121
+
122
+ def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
123
+ if x.size(0) == L:
124
+ return x
125
+ out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
126
+ out[-x.size(0):] = x
127
+ return out
128
+
129
+
130
+ def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
131
+ """通用媒体路径解析"""
132
+ p = sample.get(f"{media_type}_abs_path")
133
+ if p is None:
134
+ p = sample.get(media_type) or sample.get(f"{media_type}_path")
135
+ if p is None:
136
+ return None
137
+ if not os.path.isabs(p) and dataset_root:
138
+ p = os.path.join(dataset_root, p)
139
+ try:
140
+ return os.path.abspath(p)
141
+ except Exception:
142
+ return p
143
+
144
+
145
+ class SFTVariantDatasetLLADAV(Dataset):
146
+
147
+ def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int):
148
+ super().__init__()
149
+ self.samples = samples
150
+ self.method = method
151
+ self.lower = lower
152
+ self.upper = upper
153
+ self.block_size = block_size
154
+ self.mask_times = mask_times
155
+ self.max_gen_length = max_gen_length
156
+
157
+ self.variants: List[dict] = []
158
+ self._build_variants()
159
+
160
+ def _build_variants(self):
161
+ sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
162
+ question_prompt="\nOnly give the best option."
163
+
164
+ rng = np.random.default_rng(seed=42)
165
+ for sid, s in enumerate(self.samples):
166
+ if self.method == "semi-ar":
167
+ step_map = s.get("step_map", None)
168
+ if step_map is None:
169
+ step_map = list(range(self.max_gen_length))
170
+ order_list = collapse_k_unique(step_map, self.block_size)
171
+ uniq_steps = sorted(set(order_list))
172
+ for st in uniq_steps:
173
+ self.variants.append(dict(
174
+ sample_id=sid,
175
+ prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt,
176
+ response=s["response"],
177
+ image_abs_path=s.get("image_abs_path", None),
178
+ image=s.get("image", None),
179
+ image_path=s.get("image_path", None),
180
+ video_abs_path=s.get("video_abs_path", None),
181
+ video=s.get("video", None),
182
+ video_path=s.get("video_path", None),
183
+ var_type="semi_ar",
184
+ var_param=int(st),
185
+ step_map=order_list,
186
+ mask_seed=None,
187
+ ))
188
+ elif self.method == "random_masking":
189
+ for _ in range(self.mask_times):
190
+ t = float(rng.uniform(self.lower, self.upper))
191
+ mask_seed = int(rng.integers(0, 2**31 - 1))
192
+ self.variants.append(dict(
193
+ sample_id=sid,
194
+ prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt,
195
+ response=s["response"],
196
+ image_abs_path=s.get("image_abs_path", None),
197
+ image=s.get("image", None),
198
+ image_path=s.get("image_path", None),
199
+ video_abs_path=s.get("video_abs_path", None),
200
+ video=s.get("video", None),
201
+ video_path=s.get("video_path", None),
202
+ var_type="random_masking",
203
+ var_param=t,
204
+ step_map=None,
205
+ mask_seed=mask_seed,
206
+ ))
207
+ else:
208
+ raise ValueError(f"Unknown SFT method: {self.method}")
209
+
210
+ def __len__(self):
211
+ return len(self.variants)
212
+
213
+ def __getitem__(self, idx):
214
+ v = self.variants[idx]
215
+ return {
216
+ "idx": idx,
217
+ "sample_id": v["sample_id"],
218
+ "prompt": v["prompt"],
219
+ "response": v["response"],
220
+ "image_abs_path": v.get("image_abs_path", None),
221
+ "image": v.get("image", None),
222
+ "image_path": v.get("image_path", None),
223
+ "video_abs_path": v.get("video_abs_path", None),
224
+ "video": v.get("video", None),
225
+ "video_path": v.get("video_path", None),
226
+ "var_type": v["var_type"],
227
+ "var_param": v["var_param"],
228
+ "step_map": v.get("step_map", None),
229
+ "mask_seed": v.get("mask_seed", None),
230
+ }
231
+
232
+
233
+ def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
234
+ out: Dict[str, List[Any]] = {}
235
+ keys = batch[0].keys()
236
+ for k in keys:
237
+ out[k] = [b[k] for b in batch]
238
+ return out
239
+
240
+
241
+ @torch.no_grad()
242
+ def prepare_batch_noisy_embeds(
243
+ accelerator,
244
+ model,
245
+ tokenizer,
246
+ image_processor,
247
+ batch: Dict[str, List[Any]],
248
+ mask_id: int,
249
+ dataset_media_root: str | None,
250
+ ) -> tuple | None:
251
+
252
+ device = accelerator.device
253
+ bsz = len(batch["idx"])
254
+
255
+ group: Dict[int, List[int]] = {}
256
+ for i, sid in enumerate(batch["sample_id"]):
257
+ group.setdefault(int(sid), []).append(i)
258
+
259
+ sample_cache: Dict[int, Dict[str, Any]] = {}
260
+ for sid, pos_list in group.items():
261
+ first_pos = pos_list[0]
262
+ prompt_text = batch["prompt"][first_pos]
263
+ response_text = batch["response"][first_pos]
264
+
265
+ sample_stub = {
266
+ "image_abs_path": batch.get("image_abs_path", [None])[first_pos],
267
+ "image": batch.get("image", [None])[first_pos],
268
+ "image_path": batch.get("image_path", [None])[first_pos],
269
+ "video_abs_path": batch.get("video_abs_path", [None])[first_pos],
270
+ "video": batch.get("video", [None])[first_pos],
271
+ "video_path": batch.get("video_path", [None])[first_pos],
272
+ }
273
+
274
+ # 优先检测视频,其次检测图片
275
+ media_type = None
276
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
277
+ if p_abs is not None:
278
+ media_type = "video"
279
+ else:
280
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
281
+ if p_abs is not None:
282
+ media_type = "image"
283
+
284
+ img_list, image_sizes, modalities = None, None, []
285
+ if p_abs is not None:
286
+
287
+ img_list, image_sizes, video_stats = load_and_process_media(
288
+ p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
289
+ )
290
+
291
+ modalities = [media_type]
292
+
293
+ # 若包含视频且需附加时间指引,将参数注入 prompt
294
+ if media_type == "video" and getattr(model.config, "add_time_instruction", False):
295
+ video_time, frame_time, num_frames = video_stats
296
+ time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
297
+ # 剔除原有可能存在的图片 Token 以免重叠
298
+ clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
299
+ prompt_text = f"{time_instruction}\n{clean_prompt}"
300
+
301
+ # Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN)
302
+ input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
303
+
304
+ end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
305
+ seq_len = len(end_header_seq)
306
+ pos_eh = -1
307
+ if seq_len > 0 and input_ids_text.numel() >= seq_len:
308
+ ids_list = input_ids_text.tolist()
309
+ for i in range(len(ids_list) - seq_len, -1, -1):
310
+ if ids_list[i:i + seq_len] == end_header_seq:
311
+ pos_eh = i + seq_len - 1
312
+ break
313
+ assert pos_eh != -1, " <|end_header_id|> not found"
314
+ start_pos = pos_eh + 1
315
+
316
+ labels_text = input_ids_text.clone()
317
+ labels_text[:start_pos] = -100
318
+
319
+ (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
320
+ input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
321
+ )
322
+ clean_embeds = clean_embeds.squeeze(0) # (L, D)
323
+ new_labels = new_labels.squeeze(0) # (L,)
324
+
325
+ eot_id = 126348
326
+ eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
327
+ if eot_indices.numel() > 0:
328
+ last_eot_pos = eot_indices[-1, 0]
329
+ new_labels[last_eot_pos:] = -100
330
+ asst_mask = (new_labels != -100)
331
+ asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,)
332
+
333
+ sample_cache[sid] = dict(
334
+ clean_embeds=clean_embeds,
335
+ new_labels=new_labels,
336
+ asst_idx=asst_idx,
337
+ )
338
+
339
+ lengths = []
340
+ for i in range(bsz):
341
+ sid = int(batch["sample_id"][i])
342
+ lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0]))
343
+ Lmax = max(lengths)
344
+ D = next(iter(sample_cache.values()))["clean_embeds"].shape[1]
345
+
346
+ mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
347
+
348
+ noisy_list = []
349
+ labels_list = []
350
+ p_mask_list = []
351
+ attn_list = []
352
+
353
+ for i in range(bsz):
354
+ sid = int(batch["sample_id"][i])
355
+ var_type = batch["var_type"][i]
356
+ var_param = batch["var_param"][i]
357
+ step_map = batch["step_map"][i]
358
+
359
+ clean_embeds = sample_cache[sid]["clean_embeds"].clone()
360
+ new_labels = sample_cache[sid]["new_labels"].clone()
361
+ asst_idx = sample_cache[sid]["asst_idx"]
362
+
363
+ L = clean_embeds.size(0)
364
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
365
+
366
+ if var_type == "semi_ar":
367
+ order = torch.as_tensor(step_map, device=device)
368
+ M = asst_idx.numel()
369
+ if order.numel() != M:
370
+ m = min(M, order.numel())
371
+ asst_idx = asst_idx[:m]
372
+ order = order[:m]
373
+ order_full = torch.full((L,), -1, dtype=torch.long, device=device)
374
+ order_full[asst_idx] = order
375
+ step_val = int(var_param)
376
+
377
+ p = torch.empty(L, device=device).uniform_(0.0, 1.0)
378
+ lower = 0.0 + 1e-6
379
+ upper = 1.0 - 1e-6
380
+ thresh = (upper - lower) * torch.rand(1, device=device) + lower
381
+ block_mask = (order_full == step_val) & (p < thresh)
382
+
383
+ mask_pos = (order_full > step_val) | block_mask
384
+ noisy_embeds = clean_embeds.clone()
385
+ noisy_embeds[mask_pos] = mask_embed
386
+
387
+ pmask = block_mask
388
+
389
+ elif var_type == "random_masking":
390
+ M = asst_idx.numel()
391
+ if M == 0:
392
+ noisy_embeds = clean_embeds
393
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
394
+ else:
395
+ t = float(var_param)
396
+ mask_seed = batch.get("mask_seed", [None] * bsz)[i]
397
+ if mask_seed is None:
398
+ mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
399
+ gen = torch.Generator(device="cpu")
400
+ gen.manual_seed(int(mask_seed))
401
+ rnd_cpu = torch.rand(M, generator=gen) < t
402
+ rnd = rnd_cpu.to(device=device)
403
+
404
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
405
+ pmask[asst_idx] = rnd
406
+
407
+ noisy_embeds = clean_embeds.clone()
408
+ noisy_embeds[pmask] = mask_embed
409
+
410
+ else:
411
+ continue
412
+
413
+ if not pmask.any():
414
+ continue
415
+ # if not pmask.any():
416
+ # # print('*****no token masked*****')
417
+ # if asst_idx.numel() > 0:
418
+ # # 强制 Mask 最后一个 assistant token
419
+ # force_idx = asst_idx[-1]
420
+ # pmask[force_idx] = True
421
+ # noisy_embeds[force_idx] = mask_embed
422
+ # else:
423
+ # print("*********bad data*********")
424
+ # print(batch["sample_id"][i])
425
+ # # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过
426
+ # continue
427
+
428
+
429
+ noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
430
+ labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
431
+ p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
432
+ attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
433
+ attn[-L:] = True
434
+ attn_list.append(attn)
435
+
436
+ if len(noisy_list) == 0:
437
+ return None
438
+
439
+ noisy_batch = torch.stack(noisy_list, dim=0)
440
+ labels_batch = torch.stack(labels_list, dim=0)
441
+ p_mask_batch = torch.stack(p_mask_list, dim=0)
442
+ attn_batch = torch.stack(attn_list, dim=0)
443
+ return noisy_batch, labels_batch, p_mask_batch, attn_batch
444
+
445
+ # ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ...
446
+ def save_checkpoint(model, tokenizer, config, accelerator, name):
447
+ output_dir = Path(config.experiment.project)
448
+ output_dir.mkdir(parents=True, exist_ok=True)
449
+
450
+ checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
451
+ if accelerator.is_main_process and checkpoints_total_limit is not None:
452
+ ckpts = sorted(
453
+ [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
454
+ key=lambda p: int(p.name.split("-")[1]),
455
+ )
456
+ if len(ckpts) >= checkpoints_total_limit:
457
+ to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
458
+ logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
459
+ for p in to_remove:
460
+ shutil.rmtree(p, ignore_errors=True)
461
+
462
+ save_base = output_dir / "ckpt"
463
+ save_base.mkdir(exist_ok=True)
464
+
465
+ model_to_save = accelerator.unwrap_model(model)
466
+ state_dict = accelerator.get_state_dict(model)
467
+
468
+ if accelerator.is_main_process:
469
+ model_to_save.save_pretrained(
470
+ save_base / name,
471
+ save_function=accelerator.save,
472
+ state_dict=state_dict,
473
+ safe_serialization=True,
474
+ )
475
+ tokenizer.save_pretrained(str(save_base / name))
476
+
477
+ metadata = {
478
+ "save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
479
+ }
480
+ with (save_base / "metadata.json").open("w") as f:
481
+ json.dump(metadata, f, indent=2)
482
+
483
+ logger.info(f"Saved model + tokenizer to {save_base / name}")
484
+
485
+
486
+ def main():
487
+ config = get_config()
488
+
489
+ # TF32
490
+ if config.training.enable_tf32:
491
+ torch.backends.cuda.matmul.allow_tf32 = True
492
+ torch.backends.cudnn.benchmark = True
493
+ torch.backends.cudnn.deterministic = False
494
+
495
+ config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
496
+
497
+ # 🌟 实例化 DDP 处理器,允许存在未使用的参数
498
+ # ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
499
+ # process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600))
500
+
501
+ accelerator = Accelerator(
502
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
503
+ mixed_precision=config.training.mixed_precision,
504
+ log_with="wandb",
505
+ project_dir=config.experiment.logging_dir,
506
+ split_batches=True,
507
+ # kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去
508
+ # kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去
509
+ )
510
+
511
+ logging.basicConfig(
512
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
513
+ datefmt="%m/%d/%Y %H:%M:%S",
514
+ level=logging.INFO,
515
+ )
516
+ logger.info(accelerator.state, main_process_only=False)
517
+ if accelerator.is_local_main_process:
518
+ set_verbosity_info()
519
+ else:
520
+ set_verbosity_error()
521
+
522
+ if accelerator.is_main_process:
523
+ resume_wandb_run = config.wandb.resume
524
+ run_id = config.wandb.get("run_id", None)
525
+ if run_id is None:
526
+ resume_wandb_run = False
527
+ run_id = wandb.util.generate_id()
528
+ config.wandb.run_id = run_id
529
+ wandb_init_kwargs = dict(
530
+ name=config.experiment.project,
531
+ id=run_id,
532
+ resume=resume_wandb_run,
533
+ entity=config.wandb.get("entity", None),
534
+ config_exclude_keys=[],
535
+ )
536
+ wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
537
+ wandb_config.pop("experiment.resume_from_checkpoint", None)
538
+
539
+ accelerator.init_trackers(
540
+ config.experiment.project,
541
+ config=wandb_config,
542
+ init_kwargs={"wandb": wandb_init_kwargs},
543
+ )
544
+
545
+ if accelerator.is_main_process:
546
+ os.makedirs(config.experiment.project, exist_ok=True)
547
+ config_path = Path(config.experiment.project) / "config.yaml"
548
+ logging.info(f"Saving config to {config_path}")
549
+ OmegaConf.save(config, config_path)
550
+
551
+ if config.training.seed is not None:
552
+ set_seed(config.training.seed)
553
+
554
+ pretrained_model = config.model.pretrained_model
555
+ tokenizer, model, image_processor, _ = load_pretrained_model(
556
+ pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
557
+ )
558
+ model = model.to(accelerator.device)
559
+
560
+ special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
561
+ num_new = tokenizer.add_special_tokens(special_tokens)
562
+ if num_new > 0:
563
+ model.resize_token_embeddings(len(tokenizer))
564
+ if tokenizer.pad_token is None:
565
+ tokenizer.pad_token = tokenizer.eos_token
566
+
567
+ if getattr(config.training, "gradient_checkpointing_enable", False):
568
+ try:
569
+ model.enable_input_require_grads()
570
+ except Exception:
571
+ pass
572
+ try:
573
+ model.gradient_checkpointing_enable()
574
+ except Exception:
575
+ pass
576
+
577
+ mask_id = 126336
578
+
579
+ with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f:
580
+ data_list = json.load(f)
581
+
582
+ ds = SFTVariantDatasetLLADAV(
583
+ samples=data_list,
584
+ method=config.training.method,
585
+ lower=config.training.lower_p,
586
+ upper=config.training.upper_p,
587
+ block_size=config.training.block_size,
588
+ mask_times=config.training.mask_times_per_sample,
589
+ max_gen_length=config.training.max_gen_length
590
+ )
591
+
592
+ # Scheduler & DataLoader
593
+ total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
594
+ num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm)
595
+ num_train_epochs = config.training.num_train_epochs
596
+ max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
597
+
598
+ optimizer_config = config.optimizer.params
599
+ no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
600
+ optimizer_grouped_parameters = [
601
+ {
602
+ "params": [p for n, p in model.named_parameters() if
603
+ p.requires_grad and not any(nd in n for nd in no_decay)],
604
+ "weight_decay": optimizer_config.weight_decay,
605
+ },
606
+ {
607
+ "params": [p for n, p in model.named_parameters() if
608
+ p.requires_grad and any(nd in n for nd in no_decay)],
609
+ "weight_decay": 0.0,
610
+ },
611
+ ]
612
+ optimizer = AdamW(
613
+ optimizer_grouped_parameters,
614
+ lr=optimizer_config.learning_rate,
615
+ betas=(optimizer_config.beta1, optimizer_config.beta2),
616
+ weight_decay=optimizer_config.weight_decay,
617
+ eps=optimizer_config.epsilon,
618
+ )
619
+
620
+ lr_scheduler = get_scheduler(
621
+ config.lr_scheduler.scheduler,
622
+ optimizer=optimizer,
623
+ num_training_steps=max_train_steps,
624
+ num_warmup_steps=config.lr_scheduler.params.warmup_steps,
625
+ min_lr_scale=config.lr_scheduler.params.min_lr_scale
626
+ )
627
+
628
+ dl = DataLoader(
629
+ ds,
630
+ batch_size=config.training.batch_size_lm,
631
+ sampler=None,
632
+ collate_fn=collate_variants,
633
+ num_workers=0,
634
+ )
635
+
636
+ model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl)
637
+
638
+ logger.info("***** Running LLaDA-V SFT training *****")
639
+ logger.info(f" Num variants = {len(ds)}")
640
+ logger.info(f" Num training steps = {max_train_steps}")
641
+ logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
642
+ logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
643
+ logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
644
+
645
+ data_time_m = AverageMeter()
646
+ end = time.time()
647
+
648
+ from tqdm.auto import tqdm
649
+
650
+ def forward_process_batch(batch):
651
+ out = prepare_batch_noisy_embeds(
652
+ accelerator=accelerator,
653
+ model=model,
654
+ tokenizer=tokenizer,
655
+ image_processor=image_processor,
656
+ batch=batch,
657
+ mask_id=mask_id,
658
+ dataset_media_root=config.dataset.get("image_root", None),
659
+ )
660
+ if out is None:
661
+ return None
662
+ noisy_embeds, labels, p_mask, attn = out
663
+
664
+ outputs = model.get_model()(
665
+ inputs_embeds=noisy_embeds,
666
+ attention_mask=attn,
667
+ use_cache=False,
668
+ return_dict=True,
669
+ )
670
+ logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V)
671
+
672
+ # logits = outputs.logits.float() # (B, L, V)
673
+ log_probs = F.log_softmax(logits, dim=-1)
674
+
675
+ safe_labels = labels.clone()
676
+ safe_labels[labels == -100] = 0
677
+ logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L)
678
+
679
+ loss_tok = - (logp_tok * p_mask).sum(dim=1)
680
+ num_mask = torch.clamp(p_mask.sum(dim=1), min=1)
681
+ loss = (loss_tok / num_mask).mean()
682
+ return loss
683
+
684
+ first_epoch = 0
685
+ grad_accum = config.training.gradient_accumulation_steps
686
+
687
+ for epoch in range(first_epoch, num_train_epochs):
688
+ model.train()
689
+ progress_bar = tqdm(
690
+ dl,
691
+ desc=f"Epoch {epoch+1}/{num_train_epochs}",
692
+ disable=not accelerator.is_local_main_process,
693
+ dynamic_ncols=True,
694
+ leave=True
695
+ )
696
+
697
+ # for step, batch in enumerate(progress_bar, start=1):
698
+ # data_time_m.update(time.time() - end)
699
+
700
+ # # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器
701
+ # with accelerator.accumulate(model):
702
+ # # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理
703
+ # loss = forward_process_batch(batch)
704
+
705
+ # accelerator.backward(loss)
706
+
707
+ # # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪
708
+ # if accelerator.sync_gradients:
709
+ # if config.training.max_grad_norm is not None:
710
+ # accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
711
+
712
+ # optimizer.step()
713
+ # lr_scheduler.step()
714
+ # optimizer.zero_grad(set_to_none=True)
715
+
716
+ # # 警告:坚决删掉这里的 torch.cuda.empty_cache()
717
+
718
+ step = 0
719
+ for step, batch in enumerate(progress_bar, start=1):
720
+ data_time_m.update(time.time() - end)
721
+
722
+ loss = forward_process_batch(batch)
723
+ if loss is None:
724
+ continue
725
+ loss = loss / grad_accum
726
+ accelerator.backward(loss)
727
+
728
+ if step % grad_accum == 0:
729
+ if config.training.max_grad_norm is not None:
730
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
731
+ optimizer.step()
732
+ lr_scheduler.step()
733
+ optimizer.zero_grad(set_to_none=True)
734
+ torch.cuda.empty_cache()
735
+
736
+ if step % grad_accum != 0:
737
+ if config.training.max_grad_norm is not None:
738
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
739
+ optimizer.step()
740
+ lr_scheduler.step()
741
+ optimizer.zero_grad(set_to_none=True)
742
+ torch.cuda.empty_cache()
743
+
744
+ accelerator.wait_for_everyone()
745
+ save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
746
+ accelerator.end_training()
747
+
748
+
749
+ if __name__ == "__main__":
750
+ main()
wandb/run-20260319_223122-agxw5i56/files/diff.patch ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
2
+ index 0ca6138..4c35ee9 100644
3
+ --- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
4
+ +++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
5
+ @@ -10,7 +10,7 @@ downcast_bf16: 'no'
6
+ main_training_function: main
7
+ mixed_precision: bf16
8
+ num_machines: 1
9
+ -num_processes: 8
10
+ +num_processes: 4
11
+ rdzv_backend: static
12
+ same_network: true
13
+ tpu_env: []
14
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
15
+ index 3e07fbd..337824b 100644
16
+ --- a/configs/lladav_eval.yaml
17
+ +++ b/configs/lladav_eval.yaml
18
+ @@ -7,17 +7,17 @@ system:
19
+ HTTP_PROXY: null
20
+ HF_HOME: null
21
+ env_name: ""
22
+ - envs_dir: "/abs/path/of/envs"
23
+ - base_dir: "/abs/path/of/dLLM-RL"
24
+ + envs_dir: null
25
+ + base_dir: "/data4/runqing/dLLM-RL"
26
+
27
+ -model: "/abs/path/of/model"
28
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
29
+ model_base: "lladav"
30
+ answer_must_in_box: True
31
+
32
+ dataset:
33
+ eval_dataset: "SEEDBench_IMG_32"
34
+ data_type: "mmu"
35
+ - image_root: "/abs/path/of/data/images"
36
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
37
+
38
+ execute:
39
+ num_chunk: 128
40
+ @@ -35,4 +35,3 @@ rollout:
41
+ further_horizon: 128
42
+ output_unmasking_history: True
43
+ use_cache: True
44
+ -
45
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
46
+ index 7e38050..a07f620 100644
47
+ --- a/configs/rl_lladav.yaml
48
+ +++ b/configs/rl_lladav.yaml
49
+ @@ -5,9 +5,9 @@ wandb:
50
+ system:
51
+ HTTP_PROXY: null
52
+ HF_HOME: null
53
+ - env_name: "env_name"
54
+ - envs_dir: "/abs/path/of/envs"
55
+ - base_dir: "/abs/path/of/dLLM-RL"
56
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
57
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
58
+ + base_dir: "/data4/runqing/dLLM-RL"
59
+
60
+ experiment:
61
+ project: "rl_lladav"
62
+ @@ -22,17 +22,17 @@ experiment:
63
+ node_index: 0
64
+
65
+ model:
66
+ - pretrained_model: "/abs/path/of/sft-ckpt"
67
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
68
+ optimized_name: "optimized"
69
+ model_base: "lladav"
70
+ - vq_model_path: "/abs/path/of/magvitv2"
71
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
72
+ image_resolution: 512
73
+
74
+ dataset:
75
+ train_dataset: "SEEDBench_IMG_32"
76
+ optimization_data: "rl_data"
77
+ data_type: "mmu"
78
+ - image_root: "/abs/path/of/data/images"
79
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
80
+
81
+ rollout:
82
+ num_task_per_step: 128
83
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
84
+ index 6557210..001a6e2 100644
85
+ --- a/configs/sft_lladav.yaml
86
+ +++ b/configs/sft_lladav.yaml
87
+ @@ -7,13 +7,13 @@ experiment:
88
+ num_node: 1
89
+
90
+ model:
91
+ - pretrained_model: "/abs/path/of/model"
92
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
93
+ optimized_name: "optimized"
94
+
95
+ dataset:
96
+ - optimization_data: "sft_test" # ./data/sft_test.json
97
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
98
+ data_type: "mmu"
99
+ - image_root: "/base_dir/data/images"
100
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
101
+
102
+ training:
103
+ gradient_checkpointing_enable: False
104
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
105
+ index e0b183f..7a08a39 100644
106
+ --- a/data/preprocess_sft.ipynb
107
+ +++ b/data/preprocess_sft.ipynb
108
+ @@ -30,15 +30,15 @@
109
+ "#eos_token = \"<|im_end|>\"\n",
110
+ "\n",
111
+ "# llada and mmada:\n",
112
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
113
+ - "#eos_token = \"<|eot_id|>\"\n",
114
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
115
+ + "eos_token = \"<|eot_id|>\"\n",
116
+ "\n",
117
+ "# trado and sdar:\n",
118
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
119
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
120
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
121
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
122
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
123
+ - "eos_token = \"<|im_end|>\"\n",
124
+ + "# eos_token = \"<|im_end|>\"\n",
125
+ "\n",
126
+ "\n",
127
+ "def get_prompt(data_i):\n",
128
+ @@ -67,6 +67,46 @@
129
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
130
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
131
+ ]
132
+ + },
133
+ + {
134
+ + "cell_type": "code",
135
+ + "execution_count": 2,
136
+ + "id": "32bbd872",
137
+ + "metadata": {},
138
+ + "outputs": [
139
+ + {
140
+ + "name": "stdout",
141
+ + "output_type": "stream",
142
+ + "text": [
143
+ + "100\n"
144
+ + ]
145
+ + }
146
+ + ],
147
+ + "source": [
148
+ + "import json\n",
149
+ + "\n",
150
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
151
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
152
+ + "save_data_list = []\n",
153
+ + "save_num = 0\n",
154
+ + "with open(data_path, 'r') as fin, \\\n",
155
+ + " open(save_path, 'w') as fout:\n",
156
+ + " raw_data = json.load(fin)\n",
157
+ + " for data in raw_data:\n",
158
+ + " if save_num < 100:\n",
159
+ + " save_data_list.append(data)\n",
160
+ + " save_num += 1\n",
161
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
162
+ + " print(len(save_data_list))\n"
163
+ + ]
164
+ + },
165
+ + {
166
+ + "cell_type": "code",
167
+ + "execution_count": null,
168
+ + "id": "fdfde30f",
169
+ + "metadata": {},
170
+ + "outputs": [],
171
+ + "source": []
172
+ }
173
+ ],
174
+ "metadata": {
175
+ @@ -85,7 +125,7 @@
176
+ "name": "python",
177
+ "nbconvert_exporter": "python",
178
+ "pygments_lexer": "ipython3",
179
+ - "version": "3.10.18"
180
+ + "version": "3.10.19"
181
+ }
182
+ },
183
+ "nbformat": 4,
184
+ diff --git a/rl_v.py b/rl_v.py
185
+ index 8f2c0f1..79d2e5b 100644
186
+ --- a/rl_v.py
187
+ +++ b/rl_v.py
188
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
189
+ def env_prefix(env) -> str:
190
+ return (
191
+ "source ~/.bashrc && "
192
+ - f"source activate {env} && "
193
+ + "source /data8/runqing/miniconda/bin/activate && "
194
+ + "conda init && "
195
+ + f"conda activate {env} && "
196
+ )
197
+
198
+ config = get_config()
199
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
200
+ index 4068fc8..d284a23 100644
201
+ --- a/sample/llava/conversation.py
202
+ +++ b/sample/llava/conversation.py
203
+ @@ -99,7 +99,7 @@ class Conversation:
204
+ if self.version == "llama_v3":
205
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
206
+ elif self.version == "llava_llada":
207
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
208
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
209
+ else:
210
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
211
+ chat_template_messages = [{"role": "system", "content": self.system}]
212
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
213
+ index 3db5302..96e33b5 100644
214
+ --- a/sample/llava/model/builder.py
215
+ +++ b/sample/llava/model/builder.py
216
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
217
+ elif "llada" in model_name.lower():
218
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
219
+
220
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
221
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
222
+ +
223
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
224
+ if customized_config is None:
225
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
226
+ else:
227
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
228
+ index 4068fc8..d284a23 100644
229
+ --- a/train/llava/conversation.py
230
+ +++ b/train/llava/conversation.py
231
+ @@ -99,7 +99,7 @@ class Conversation:
232
+ if self.version == "llama_v3":
233
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
234
+ elif self.version == "llava_llada":
235
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
236
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
237
+ else:
238
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
239
+ chat_template_messages = [{"role": "system", "content": self.system}]
240
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
241
+ index 3db5302..aafab95 100644
242
+ --- a/train/llava/model/builder.py
243
+ +++ b/train/llava/model/builder.py
244
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
245
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
246
+ elif "llada" in model_name.lower():
247
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
248
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
249
+
250
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
251
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
252
+ if customized_config is None:
253
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
254
+ else:
255
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
256
+ index 3da28dc..8b17da3 100644
257
+ --- a/train/llava/model/language_model/llava_llada.py
258
+ +++ b/train/llava/model/language_model/llava_llada.py
259
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
260
+ dpo_forward: Optional[bool] = None,
261
+ cache_position=None,
262
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
263
+ -
264
+ + conversation_ids = None
265
+ if inputs_embeds is None and attention_mask is not None:
266
+ # donate multi-dialogue
267
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
2
+ index 0ca6138..4c35ee9 100644
3
+ --- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
4
+ +++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml
5
+ @@ -10,7 +10,7 @@ downcast_bf16: 'no'
6
+ main_training_function: main
7
+ mixed_precision: bf16
8
+ num_machines: 1
9
+ -num_processes: 8
10
+ +num_processes: 4
11
+ rdzv_backend: static
12
+ same_network: true
13
+ tpu_env: []
14
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
15
+ index 3e07fbd..337824b 100644
16
+ --- a/configs/lladav_eval.yaml
17
+ +++ b/configs/lladav_eval.yaml
18
+ @@ -7,17 +7,17 @@ system:
19
+ HTTP_PROXY: null
20
+ HF_HOME: null
21
+ env_name: ""
22
+ - envs_dir: "/abs/path/of/envs"
23
+ - base_dir: "/abs/path/of/dLLM-RL"
24
+ + envs_dir: null
25
+ + base_dir: "/data4/runqing/dLLM-RL"
26
+
27
+ -model: "/abs/path/of/model"
28
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
29
+ model_base: "lladav"
30
+ answer_must_in_box: True
31
+
32
+ dataset:
33
+ eval_dataset: "SEEDBench_IMG_32"
34
+ data_type: "mmu"
35
+ - image_root: "/abs/path/of/data/images"
36
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
37
+
38
+ execute:
39
+ num_chunk: 128
40
+ @@ -35,4 +35,3 @@ rollout:
41
+ further_horizon: 128
42
+ output_unmasking_history: True
43
+ use_cache: True
44
+ -
45
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
46
+ index 7e38050..a07f620 100644
47
+ --- a/configs/rl_lladav.yaml
48
+ +++ b/configs/rl_lladav.yaml
49
+ @@ -5,9 +5,9 @@ wandb:
50
+ system:
51
+ HTTP_PROXY: null
52
+ HF_HOME: null
53
+ - env_name: "env_name"
54
+ - envs_dir: "/abs/path/of/envs"
55
+ - base_dir: "/abs/path/of/dLLM-RL"
56
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
57
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
58
+ + base_dir: "/data4/runqing/dLLM-RL"
59
+
60
+ experiment:
61
+ project: "rl_lladav"
62
+ @@ -22,17 +22,17 @@ experiment:
63
+ node_index: 0
64
+
65
+ model:
66
+ - pretrained_model: "/abs/path/of/sft-ckpt"
67
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
68
+ optimized_name: "optimized"
69
+ model_base: "lladav"
70
+ - vq_model_path: "/abs/path/of/magvitv2"
71
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
72
+ image_resolution: 512
73
+
74
+ dataset:
75
+ train_dataset: "SEEDBench_IMG_32"
76
+ optimization_data: "rl_data"
77
+ data_type: "mmu"
78
+ - image_root: "/abs/path/of/data/images"
79
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
80
+
81
+ rollout:
82
+ num_task_per_step: 128
83
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
84
+ index 6557210..001a6e2 100644
85
+ --- a/configs/sft_lladav.yaml
86
+ +++ b/configs/sft_lladav.yaml
87
+ @@ -7,13 +7,13 @@ experiment:
88
+ num_node: 1
89
+
90
+ model:
91
+ - pretrained_model: "/abs/path/of/model"
92
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
93
+ optimized_name: "optimized"
94
+
95
+ dataset:
96
+ - optimization_data: "sft_test" # ./data/sft_test.json
97
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
98
+ data_type: "mmu"
99
+ - image_root: "/base_dir/data/images"
100
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
101
+
102
+ training:
103
+ gradient_checkpointing_enable: False
104
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
105
+ index e0b183f..7a08a39 100644
106
+ --- a/data/preprocess_sft.ipynb
107
+ +++ b/data/preprocess_sft.ipynb
108
+ @@ -30,15 +30,15 @@
109
+ "#eos_token = \"<|im_end|>\"\n",
110
+ "\n",
111
+ "# llada and mmada:\n",
112
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
113
+ - "#eos_token = \"<|eot_id|>\"\n",
114
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
115
+ + "eos_token = \"<|eot_id|>\"\n",
116
+ "\n",
117
+ "# trado and sdar:\n",
118
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
119
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
120
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
121
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
122
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
123
+ - "eos_token = \"<|im_end|>\"\n",
124
+ + "# eos_token = \"<|im_end|>\"\n",
125
+ "\n",
126
+ "\n",
127
+ "def get_prompt(data_i):\n",
128
+ @@ -67,6 +67,46 @@
129
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
130
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
131
+ ]
132
+ + },
133
+ + {
134
+ + "cell_type": "code",
135
+ + "execution_count": 2,
136
+ + "id": "32bbd872",
137
+ + "metadata": {},
138
+ + "outputs": [
139
+ + {
140
+ + "name": "stdout",
141
+ + "output_type": "stream",
142
+ + "text": [
143
+ + "100\n"
144
+ + ]
145
+ + }
146
+ + ],
147
+ + "source": [
148
+ + "import json\n",
149
+ + "\n",
150
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
151
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
152
+ + "save_data_list = []\n",
153
+ + "save_num = 0\n",
154
+ + "with open(data_path, 'r') as fin, \\\n",
155
+ + " open(save_path, 'w') as fout:\n",
156
+ + " raw_data = json.load(fin)\n",
157
+ + " for data in raw_data:\n",
158
+ + " if save_num < 100:\n",
159
+ + " save_data_list.append(data)\n",
160
+ + " save_num += 1\n",
161
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
162
+ + " print(len(save_data_list))\n"
163
+ + ]
164
+ + },
165
+ + {
166
+ + "cell_type": "code",
167
+ + "execution_count": null,
168
+ + "id": "fdfde30f",
169
+ + "metadata": {},
170
+ + "outputs": [],
171
+ + "source": []
172
+ }
173
+ ],
174
+ "metadata": {
175
+ @@ -85,7 +125,7 @@
176
+ "name": "python",
177
+ "nbconvert_exporter": "python",
178
+ "pygments_lexer": "ipython3",
179
+ - "version": "3.10.18"
180
+ + "version": "3.10.19"
181
+ }
182
+ },
183
+ "nbformat": 4,
184
+ diff --git a/rl_v.py b/rl_v.py
185
+ index 8f2c0f1..79d2e5b 100644
186
+ --- a/rl_v.py
187
+ +++ b/rl_v.py
188
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
189
+ def env_prefix(env) -> str:
190
+ return (
191
+ "source ~/.bashrc && "
192
+ - f"source activate {env} && "
193
+ + "source /data8/runqing/miniconda/bin/activate && "
194
+ + "conda init && "
195
+ + f"conda activate {env} && "
196
+ )
197
+
198
+ config = get_config()
199
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
200
+ index 4068fc8..d284a23 100644
201
+ --- a/sample/llava/conversation.py
202
+ +++ b/sample/llava/conversation.py
203
+ @@ -99,7 +99,7 @@ class Conversation:
204
+ if self.version == "llama_v3":
205
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
206
+ elif self.version == "llava_llada":
207
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
208
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
209
+ else:
210
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
211
+ chat_template_messages = [{"role": "system", "content": self.system}]
212
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
213
+ index 3db5302..96e33b5 100644
214
+ --- a/sample/llava/model/builder.py
215
+ +++ b/sample/llava/model/builder.py
216
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
217
+ elif "llada" in model_name.lower():
218
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
219
+
220
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
221
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
222
+ +
223
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
224
+ if customized_config is None:
225
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
226
+ else:
227
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
228
+ index 4068fc8..d284a23 100644
229
+ --- a/train/llava/conversation.py
230
+ +++ b/train/llava/conversation.py
231
+ @@ -99,7 +99,7 @@ class Conversation:
232
+ if self.version == "llama_v3":
233
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
234
+ elif self.version == "llava_llada":
235
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
236
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
237
+ else:
238
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
239
+ chat_template_messages = [{"role": "system", "content": self.system}]
240
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
241
+ index 3db5302..aafab95 100644
242
+ --- a/train/llava/model/builder.py
243
+ +++ b/train/llava/model/builder.py
244
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
245
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
246
+ elif "llada" in model_name.lower():
247
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
248
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
249
+
250
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
251
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
252
+ if customized_config is None:
253
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
254
+ else:
255
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
256
+ index 3da28dc..8b17da3 100644
257
+ --- a/train/llava/model/language_model/llava_llada.py
258
+ +++ b/train/llava/model/language_model/llava_llada.py
259
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
260
+ dpo_forward: Optional[bool] = None,
261
+ cache_position=None,
262
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
263
+ -
264
+ + conversation_ids = None
265
+ if inputs_embeds is None and attention_mask is not None:
266
+ # donate multi-dialogue
267
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
wandb/run-20260319_223122-agxw5i56/files/output.log ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 03/19/2026 22:31:26 - INFO - root - Saving config to sft_lladav_video_reversal_3_19/config.yaml
2
+ Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/LLaDA-V
3
+ Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
4
+ Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:20<00:00, 3.37s/it]
5
+ Rank 0: Model Class: LlavaLLaDAModelLM
6
+ /data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
7
+ If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
8
+ warnings.warn(
9
+ Before initializing optimizer states
10
+ MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB
11
+ CPU Virtual Memory: used = 208.51 GB, percent = 10.3%
12
+ After initializing optimizer states
13
+ MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB
14
+ CPU Virtual Memory: used = 228.52 GB, percent = 11.3%
15
+ After initializing ZeRO optimizer
16
+ MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB
17
+ CPU Virtual Memory: used = 229.31 GB, percent = 11.4%
18
+ 03/19/2026 22:33:07 - INFO - __main__ - ***** Running LLaDA-V SFT training *****
19
+ 03/19/2026 22:33:07 - INFO - __main__ - Num variants = 95996
20
+ 03/19/2026 22:33:07 - INFO - __main__ - Num training steps = 3001
21
+ 03/19/2026 22:33:07 - INFO - __main__ - Instantaneous batch size per device = 2
22
+ 03/19/2026 22:33:07 - INFO - __main__ - Total train batch size (parallel * accum) = 32
23
+ 03/19/2026 22:33:07 - INFO - __main__ - Gradient Accumulation steps = 4
24
+ Epoch 1/1: 0%| | 0/12000 [00:00<?, ?it/s]/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/checkpoint.py:87: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
25
+ warnings.warn(
26
+ Epoch 1/1: 44%|████████████████████████████████████████████████████████▋ | 5319/12000 [20:01:24<32:33:17, 17.54s/it]
wandb/run-20260319_223122-agxw5i56/files/requirements.txt ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ transformers==4.52.4
2
+ nvidia-nvtx-cu12==12.4.127
3
+ deepspeed==0.18.6
4
+ typing-inspection==0.4.2
5
+ scipy==1.15.3
6
+ MarkupSafe==3.0.3
7
+ nvidia-cuda-runtime-cu12==12.4.127
8
+ yarl==1.22.0
9
+ tornado==6.5.4
10
+ nvidia-curand-cu12==10.3.5.147
11
+ nest-asyncio==1.6.0
12
+ safetensors==0.7.0
13
+ async-timeout==5.0.1
14
+ typing_extensions==4.15.0
15
+ asttokens==3.0.1
16
+ cuda-python==13.1.1
17
+ executing==2.2.1
18
+ nvidia-cudnn-frontend==1.18.0
19
+ comm==0.2.3
20
+ tokenizers==0.21.4
21
+ six==1.17.0
22
+ Pygments==2.19.2
23
+ nvidia-cusparselt-cu12==0.6.2
24
+ apache-tvm-ffi==0.1.8.post2
25
+ flash_attn==2.7.4.post1
26
+ tqdm==4.67.3
27
+ tensorboard-data-server==0.7.2
28
+ annotated-types==0.7.0
29
+ math-verify==0.9.0
30
+ idna==3.11
31
+ regex==2026.1.15
32
+ nvidia-cutlass-dsl-libs-base==4.4.0
33
+ flashinfer-python==0.6.3
34
+ click==8.3.1
35
+ pytz==2025.2
36
+ wheel==0.46.3
37
+ ipykernel==7.2.0
38
+ stack-data==0.6.3
39
+ absl-py==2.4.0
40
+ wandb==0.25.0
41
+ pandas==2.3.3
42
+ termcolor==3.3.0
43
+ sympy==1.13.1
44
+ joblib==1.5.3
45
+ prompt_toolkit==3.0.52
46
+ attrs==25.4.0
47
+ nvidia-cusolver-cu12==11.6.1.9
48
+ xxhash==3.6.0
49
+ aiohappyeyeballs==2.6.1
50
+ python-dateutil==2.9.0.post0
51
+ frozenlist==1.8.0
52
+ nvidia-ml-py==13.590.48
53
+ multiprocess==0.70.18
54
+ decorator==5.2.1
55
+ cuda-pathfinder==1.3.4
56
+ multidict==6.7.1
57
+ pure_eval==0.2.3
58
+ nvidia-cufft-cu12==11.2.1.3
59
+ omegaconf==2.3.0
60
+ PyYAML==6.0.3
61
+ debugpy==1.8.20
62
+ psutil==7.2.2
63
+ ninja==1.13.0
64
+ nvidia-cublas-cu12==12.4.5.8
65
+ triton==3.2.0
66
+ antlr4-python3-runtime==4.9.3
67
+ peft==0.18.1
68
+ aiosignal==1.4.0
69
+ accelerate==1.12.0
70
+ liger_kernel==0.7.0
71
+ anyio==4.12.1
72
+ decord==0.6.0
73
+ h11==0.16.0
74
+ urllib3==2.6.3
75
+ wcwidth==0.6.0
76
+ networkx==3.4.2
77
+ huggingface_hub==0.36.2
78
+ tzdata==2025.3
79
+ xformers==0.0.29.post3
80
+ tabulate==0.9.0
81
+ certifi==2026.1.4
82
+ tensorboard==2.20.0
83
+ matplotlib-inline==0.2.1
84
+ pip==26.0.1
85
+ smmap==5.0.2
86
+ nvidia-cuda-cupti-cu12==12.4.127
87
+ Markdown==3.10.2
88
+ nvidia-nccl-cu12==2.21.5
89
+ aiohttp==3.13.3
90
+ scikit-learn==1.7.2
91
+ nvidia-cutlass-dsl==4.4.0
92
+ setuptools==82.0.0
93
+ jupyter_core==5.9.1
94
+ asyncio==4.0.0
95
+ einops==0.8.2
96
+ pydantic==2.12.5
97
+ av==16.1.0
98
+ protobuf==6.33.5
99
+ evaluate==0.4.6
100
+ Jinja2==3.1.6
101
+ exceptiongroup==1.3.1
102
+ ipython==8.38.0
103
+ sentry-sdk==2.53.0
104
+ httpx==0.28.1
105
+ msgpack==1.1.2
106
+ platformdirs==4.9.2
107
+ hjson==3.1.0
108
+ torchvision==0.21.0
109
+ parso==0.8.6
110
+ cuda-bindings==13.1.1
111
+ pexpect==4.9.0
112
+ threadpoolctl==3.6.0
113
+ pyarrow==23.0.1
114
+ jedi==0.19.2
115
+ nvidia-cuda-nvrtc-cu12==12.4.127
116
+ Werkzeug==3.1.5
117
+ fsspec==2025.10.0
118
+ pydantic_core==2.41.5
119
+ GitPython==3.1.46
120
+ propcache==0.4.1
121
+ mpmath==1.3.0
122
+ nvidia-nvjitlink-cu12==12.4.127
123
+ nvidia-cusparse-cu12==12.3.1.170
124
+ hf-xet==1.2.0
125
+ charset-normalizer==3.4.4
126
+ dill==0.4.0
127
+ datasets==4.5.0
128
+ py-cpuinfo==9.0.0
129
+ sentencepiece==0.2.1
130
+ latex2sympy2_extended==1.11.0
131
+ jupyter_client==8.8.0
132
+ pyzmq==27.1.0
133
+ numpy==2.2.6
134
+ gitdb==4.0.12
135
+ grpcio==1.78.0
136
+ packaging==26.0
137
+ requests==2.32.5
138
+ nvidia-cudnn-cu12==9.1.0.70
139
+ httpcore==1.0.9
140
+ traitlets==5.14.3
141
+ filelock==3.24.2
142
+ ptyprocess==0.7.0
143
+ pillow==12.1.1
144
+ torch==2.6.0
145
+ more-itertools==10.8.0
146
+ wheel==0.46.3
147
+ jaraco.text==4.0.0
148
+ zipp==3.23.0
149
+ autocommand==2.2.2
150
+ jaraco.context==6.1.0
151
+ jaraco.functools==4.4.0
152
+ tomli==2.4.0
153
+ backports.tarfile==1.2.0
154
+ importlib_metadata==8.7.1
155
+ platformdirs==4.4.0
156
+ packaging==26.0
wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.19",
4
+ "startedAt": "2026-03-19T14:31:22.110949Z",
5
+ "args": [
6
+ "config=configs/sft_lladav_video_reversal.yaml"
7
+ ],
8
+ "program": "/data4/runqing/dLLM-RL/train/sft_lladav_video_reversal.py",
9
+ "codePath": "train/sft_lladav_video_reversal.py",
10
+ "codePathLocal": "train/sft_lladav_video_reversal.py",
11
+ "git": {
12
+ "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
13
+ "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
14
+ },
15
+ "email": "shixi_runqing@staff.weibo.com",
16
+ "root": "/data4/runqing/dLLM-RL",
17
+ "host": "b1301b38c75f",
18
+ "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
19
+ "cpu_count": 96,
20
+ "cpu_count_logical": 192,
21
+ "gpu": "NVIDIA H800",
22
+ "gpu_count": 8,
23
+ "disk": {
24
+ "/": {
25
+ "total": "806578724864",
26
+ "used": "570999476224"
27
+ }
28
+ },
29
+ "memory": {
30
+ "total": "2164113813504"
31
+ },
32
+ "gpu_nvidia": [
33
+ {
34
+ "name": "NVIDIA H800",
35
+ "memoryTotal": "85520809984",
36
+ "cudaCores": 16896,
37
+ "architecture": "Hopper",
38
+ "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
39
+ },
40
+ {
41
+ "name": "NVIDIA H800",
42
+ "memoryTotal": "85520809984",
43
+ "cudaCores": 16896,
44
+ "architecture": "Hopper",
45
+ "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
46
+ },
47
+ {
48
+ "name": "NVIDIA H800",
49
+ "memoryTotal": "85520809984",
50
+ "cudaCores": 16896,
51
+ "architecture": "Hopper",
52
+ "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
53
+ },
54
+ {
55
+ "name": "NVIDIA H800",
56
+ "memoryTotal": "85520809984",
57
+ "cudaCores": 16896,
58
+ "architecture": "Hopper",
59
+ "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
60
+ },
61
+ {
62
+ "name": "NVIDIA H800",
63
+ "memoryTotal": "85520809984",
64
+ "cudaCores": 16896,
65
+ "architecture": "Hopper",
66
+ "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
67
+ },
68
+ {
69
+ "name": "NVIDIA H800",
70
+ "memoryTotal": "85520809984",
71
+ "cudaCores": 16896,
72
+ "architecture": "Hopper",
73
+ "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
74
+ },
75
+ {
76
+ "name": "NVIDIA H800",
77
+ "memoryTotal": "85520809984",
78
+ "cudaCores": 16896,
79
+ "architecture": "Hopper",
80
+ "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
81
+ },
82
+ {
83
+ "name": "NVIDIA H800",
84
+ "memoryTotal": "85520809984",
85
+ "cudaCores": 16896,
86
+ "architecture": "Hopper",
87
+ "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
88
+ }
89
+ ],
90
+ "cudaVersion": "12.4",
91
+ "writerId": "e2na0by3kxag1d04jgsxtaqowuonipoi"
92
+ }
wandb/run-20260319_223122-agxw5i56/logs/debug-core.log ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-19T22:31:22.229721003+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpec9x2qfv/port-3863399.txt","pid":3863399,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-03-19T22:31:22.232304613+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3863399}
3
+ {"time":"2026-03-19T22:31:22.2322714+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3863399-3864444-838468677/socket","Net":"unix"}}
4
+ {"time":"2026-03-19T22:31:22.373663717+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-03-19T22:31:22.3828339+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"agxw5i56","id":"1(@)"}
6
+ {"time":"2026-03-19T22:31:26.60424022+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"agxw5i56","id":"1(@)"}
7
+ {"time":"2026-03-19T22:31:31.926890311+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6am2qm8cdzui"}
8
+ {"time":"2026-03-20T20:22:41.232415566+08:00","level":"INFO","msg":"server: parent process exited, terminating service process"}
wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-19T22:31:22.38308208+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
2
+ {"time":"2026-03-19T22:31:26.603517195+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
3
+ {"time":"2026-03-19T22:31:26.603666661+08:00","level":"INFO","msg":"stream: created new stream","id":"agxw5i56"}
4
+ {"time":"2026-03-19T22:31:26.604080689+08:00","level":"INFO","msg":"handler: started","stream_id":"agxw5i56"}
5
+ {"time":"2026-03-19T22:31:26.60422441+08:00","level":"INFO","msg":"stream: started","id":"agxw5i56"}
6
+ {"time":"2026-03-19T22:31:26.604248603+08:00","level":"INFO","msg":"writer: started","stream_id":"agxw5i56"}
7
+ {"time":"2026-03-19T22:31:26.604250045+08:00","level":"INFO","msg":"sender: started","stream_id":"agxw5i56"}
8
+ {"time":"2026-03-19T22:31:26.606722875+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
9
+ {"time":"2026-03-19T22:31:31.800015446+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a341b0ef13db33a9cfc8c700b1fd196501c0ae91a1f3805869f4c704a35346a3"}
10
+ {"time":"2026-03-19T22:31:31.800189317+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch</Resource><RequestId>189E4490C4AA826F</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a341b0ef13db33a9cfc8c700b1fd196501c0ae91a1f3805869f4c704a35346a3, Size: 11222}"}
11
+ {"time":"2026-03-19T22:31:31.801370376+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1bfe21a4d7c63fe36fba0cc6e7d8abc0349992391788b2eb567126807629b913"}
12
+ {"time":"2026-03-19T22:31:31.801479894+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt</Resource><RequestId>189E4490C4D2AC1E</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1bfe21a4d7c63fe36fba0cc6e7d8abc0349992391788b2eb567126807629b913, Size: 2966}"}
13
+ {"time":"2026-03-19T22:31:31.802152269+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=633c45789455deff40ac9ac290a8030f078715a583a3cc959d99dfeef370bdc2"}
14
+ {"time":"2026-03-19T22:31:31.802279633+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189E4490C44A5AF8</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=633c45789455deff40ac9ac290a8030f078715a583a3cc959d99dfeef370bdc2, Size: 11222}"}
15
+ {"time":"2026-03-19T22:31:31.803630426+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f087ddbd49f5feebf121bad34dc7ee132d5e81a79967564da3bbaee6ca6044cf"}
16
+ {"time":"2026-03-19T22:31:31.803693079+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json</Resource><RequestId>189E4490C4F3B717</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f087ddbd49f5feebf121bad34dc7ee132d5e81a79967564da3bbaee6ca6044cf, Size: 2650}"}
17
+ {"time":"2026-03-19T22:31:31.810924488+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e90cb55cb9fe21539c6e1e8d4e3349ca70bfaa57d266471b6485dd8c5fdb2bfe"}
18
+ {"time":"2026-03-19T22:31:31.810989568+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py</Resource><RequestId>189E4490C4D0F1F3</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py, Name: code/train/sft_lladav_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e90cb55cb9fe21539c6e1e8d4e3349ca70bfaa57d266471b6485dd8c5fdb2bfe, Size: 29481}"}
wandb/run-20260319_223122-agxw5i56/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
2
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Configure stats pid to 3863399
3
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/logs/debug.log
5
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log
6
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():844] calling init triggers
7
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
+ config: {'_wandb': {'code_path': 'code/train/sft_lladav_video_reversal.py'}}
9
+ 2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():892] starting backend
10
+ 2026-03-19 22:31:22,373 INFO MainThread:3863399 [wandb_init.py:init():895] sending inform_init request
11
+ 2026-03-19 22:31:22,377 INFO MainThread:3863399 [wandb_init.py:init():903] backend started and connected
12
+ 2026-03-19 22:31:22,378 INFO MainThread:3863399 [wandb_init.py:init():973] updated telemetry
13
+ 2026-03-19 22:31:22,383 INFO MainThread:3863399 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
+ 2026-03-19 22:31:26,722 INFO MainThread:3863399 [wandb_init.py:init():1042] starting run threads in backend
15
+ 2026-03-19 22:31:26,920 INFO MainThread:3863399 [wandb_run.py:_console_start():2524] atexit reg
16
+ 2026-03-19 22:31:26,920 INFO MainThread:3863399 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-03-19 22:31:26,921 INFO MainThread:3863399 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-03-19 22:31:26,921 INFO MainThread:3863399 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-03-19 22:31:26,928 INFO MainThread:3863399 [wandb_init.py:init():1082] run started, returning control to user process
20
+ 2026-03-19 22:31:26,929 INFO MainThread:3863399 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'agxw5i56', 'experiment.project': 'sft_lladav_video_reversal_3_19', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lladav_video_reversal_3_19/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 4, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.1, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lladav_video_reversal.yaml'}
wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py ADDED
@@ -0,0 +1,757 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
4
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
5
+ from datetime import timedelta
6
+ import json
7
+ import logging
8
+ import math
9
+ import shutil
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Dict, List, Any
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from torch.optim import AdamW
18
+ from torch.utils.data import Dataset, DataLoader
19
+
20
+ from omegaconf import OmegaConf
21
+ import wandb
22
+
23
+ from accelerate import Accelerator, InitProcessGroupKwargs
24
+ from accelerate.logging import get_logger
25
+ from accelerate.utils import set_seed
26
+
27
+ from train.utils import get_config, flatten_omega_conf, AverageMeter
28
+ from models.lr_schedulers import get_scheduler
29
+ from models.logging import set_verbosity_info, set_verbosity_error
30
+
31
+ # LLaDA-V / LLaVA
32
+ from transformers.utils import logging as hf_logging
33
+ hf_logging.set_verbosity_error()
34
+
35
+ from llava.model.builder import load_pretrained_model
36
+ from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord
37
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
38
+ from llava.conversation import conv_templates
39
+ # from accelerate import DistributedDataParallelKwargs #添加
40
+
41
+ logger = get_logger(__name__, log_level="INFO")
42
+
43
+
44
+ def collapse_k_unique(lst, k: int):
45
+ if k <= 0:
46
+ raise ValueError("k must be > 0")
47
+ uniq = sorted(set(lst))
48
+ mapping = {}
49
+ n = len(uniq)
50
+ for idx, val in enumerate(uniq):
51
+ group = idx // k
52
+ end_idx = min((group + 1) * k - 1, n - 1)
53
+ rep = uniq[end_idx]
54
+ mapping[val] = rep
55
+ return [mapping[x] for x in lst]
56
+
57
+
58
+ def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
59
+ import copy
60
+ conv = copy.deepcopy(conv_templates["llava_llada"])
61
+ conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
62
+ conv.append_message(conv.roles[1], response_text)
63
+ return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
64
+
65
+
66
+ def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
67
+ conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
68
+ ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
69
+ return ids.squeeze(0) # (L,)
70
+
71
+
72
+ def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
73
+ """
74
+ 统一处理 Image 和 Video
75
+ 返回 (media_tensor_list, sizes, video_stats)
76
+ """
77
+ from PIL import Image
78
+ from types import SimpleNamespace
79
+
80
+ if isinstance(model_cfg, dict):
81
+ model_cfg = SimpleNamespace(**model_cfg)
82
+
83
+ if media_type == "image":
84
+ image = Image.open(abs_path).convert("RGB")
85
+ if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
86
+ setattr(model_cfg, "image_aspect_ratio", "pad")
87
+ if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
88
+ setattr(model_cfg, "mm_patch_merge_type", "flat")
89
+
90
+ image_tensor_list = process_images([image], image_processor, model_cfg)
91
+ image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
92
+ image_sizes = [image.size]
93
+ return image_tensor_list, image_sizes, None
94
+
95
+ elif media_type == "video":
96
+ if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
97
+ if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16)
98
+ if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True)
99
+
100
+ # 使用官方提供的 decord 函数处理视频
101
+ video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
102
+
103
+ # 处理成 tensor
104
+ image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
105
+ image_tensor = image_tensor.to(dtype=dtype, device=device)
106
+
107
+ # 提取 size
108
+ if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
109
+ v_size = (video_frames.shape[2], video_frames.shape[1])
110
+ else:
111
+ v_size = (0, 0)
112
+
113
+ return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
114
+
115
+
116
+ def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
117
+ out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
118
+ out[-x.numel():] = x
119
+ return out
120
+
121
+
122
+ def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
123
+ if x.size(0) == L:
124
+ return x
125
+ out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
126
+ out[-x.size(0):] = x
127
+ return out
128
+
129
+
130
+ def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
131
+ """通用媒体路径解析"""
132
+ p = sample.get(f"{media_type}_abs_path")
133
+ if p is None:
134
+ p = sample.get(media_type) or sample.get(f"{media_type}_path")
135
+ if p is None:
136
+ return None
137
+ if not os.path.isabs(p) and dataset_root:
138
+ p = os.path.join(dataset_root, p)
139
+ try:
140
+ return os.path.abspath(p)
141
+ except Exception:
142
+ return p
143
+
144
+
145
+ class SFTVariantDatasetLLADAV(Dataset):
146
+
147
+ def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int):
148
+ super().__init__()
149
+ self.samples = samples
150
+ self.method = method
151
+ self.lower = lower
152
+ self.upper = upper
153
+ self.block_size = block_size
154
+ self.mask_times = mask_times
155
+ self.max_gen_length = max_gen_length
156
+
157
+ self.variants: List[dict] = []
158
+ self._build_variants()
159
+
160
+ def _build_variants(self):
161
+ sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
162
+ question_prompt="\nOnly give the best option."
163
+ multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
164
+
165
+ rng = np.random.default_rng(seed=42)
166
+ for sid, s in enumerate(self.samples):
167
+ if self.method == "semi-ar":
168
+ step_map = s.get("step_map", None)
169
+ if step_map is None:
170
+ step_map = list(range(self.max_gen_length))
171
+ order_list = collapse_k_unique(step_map, self.block_size)
172
+ uniq_steps = sorted(set(order_list))
173
+ for st in uniq_steps:
174
+ self.variants.append(dict(
175
+ sample_id=sid,
176
+ prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt,
177
+ response=s["response"],
178
+ image_abs_path=s.get("image_abs_path", None),
179
+ image=s.get("image", None),
180
+ image_path=s.get("image_path", None),
181
+ video_abs_path=s.get("video_abs_path", None),
182
+ video=s.get("video", None),
183
+ video_path=s.get("video_path", None),
184
+ var_type="semi_ar",
185
+ var_param=int(st),
186
+ step_map=order_list,
187
+ mask_seed=None,
188
+ ))
189
+ elif self.method == "random_masking":
190
+ for _ in range(self.mask_times):
191
+ t = float(rng.uniform(self.lower, self.upper))
192
+ mask_seed = int(rng.integers(0, 2**31 - 1))
193
+ self.variants.append(dict(
194
+ sample_id=sid,
195
+ prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt,
196
+ response=s["response"],
197
+ image_abs_path=s.get("image_abs_path", None),
198
+ image=s.get("image", None),
199
+ image_path=s.get("image_path", None),
200
+ video_abs_path=s.get("video_abs_path", None),
201
+ video=s.get("video", None),
202
+ video_path=s.get("video_path", None),
203
+ var_type="random_masking",
204
+ var_param=t,
205
+ step_map=None,
206
+ mask_seed=mask_seed,
207
+ ))
208
+ else:
209
+ raise ValueError(f"Unknown SFT method: {self.method}")
210
+
211
+ def __len__(self):
212
+ return len(self.variants)
213
+
214
+ def __getitem__(self, idx):
215
+ v = self.variants[idx]
216
+ return {
217
+ "idx": idx,
218
+ "sample_id": v["sample_id"],
219
+ "prompt": v["prompt"],
220
+ "response": v["response"],
221
+ "image_abs_path": v.get("image_abs_path", None),
222
+ "image": v.get("image", None),
223
+ "image_path": v.get("image_path", None),
224
+ "video_abs_path": v.get("video_abs_path", None),
225
+ "video": v.get("video", None),
226
+ "video_path": v.get("video_path", None),
227
+ "var_type": v["var_type"],
228
+ "var_param": v["var_param"],
229
+ "step_map": v.get("step_map", None),
230
+ "mask_seed": v.get("mask_seed", None),
231
+ }
232
+
233
+
234
+ def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
235
+ out: Dict[str, List[Any]] = {}
236
+ keys = batch[0].keys()
237
+ for k in keys:
238
+ out[k] = [b[k] for b in batch]
239
+ return out
240
+
241
+
242
+ @torch.no_grad()
243
+ def prepare_batch_noisy_embeds(
244
+ accelerator,
245
+ model,
246
+ tokenizer,
247
+ image_processor,
248
+ batch: Dict[str, List[Any]],
249
+ mask_id: int,
250
+ dataset_media_root: str | None,
251
+ ) -> tuple | None:
252
+
253
+ device = accelerator.device
254
+ bsz = len(batch["idx"])
255
+
256
+ group: Dict[int, List[int]] = {}
257
+ for i, sid in enumerate(batch["sample_id"]):
258
+ group.setdefault(int(sid), []).append(i)
259
+
260
+ sample_cache: Dict[int, Dict[str, Any]] = {}
261
+ for sid, pos_list in group.items():
262
+ first_pos = pos_list[0]
263
+ prompt_text = batch["prompt"][first_pos]
264
+ response_text = batch["response"][first_pos]
265
+
266
+ sample_stub = {
267
+ "image_abs_path": batch.get("image_abs_path", [None])[first_pos],
268
+ "image": batch.get("image", [None])[first_pos],
269
+ "image_path": batch.get("image_path", [None])[first_pos],
270
+ "video_abs_path": batch.get("video_abs_path", [None])[first_pos],
271
+ "video": batch.get("video", [None])[first_pos],
272
+ "video_path": batch.get("video_path", [None])[first_pos],
273
+ }
274
+
275
+ # 优先检测视频,其次检测图片
276
+ media_type = None
277
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
278
+ if p_abs is not None:
279
+ media_type = "video"
280
+ else:
281
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
282
+ if p_abs is not None:
283
+ media_type = "image"
284
+
285
+ img_list, image_sizes, modalities = None, None, []
286
+ if p_abs is not None:
287
+
288
+ img_list, image_sizes, video_stats = load_and_process_media(
289
+ p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
290
+ )
291
+
292
+ modalities = [media_type]
293
+
294
+ # 若包含视频且需附加时间指引,将参数注入 prompt
295
+ if media_type == "video" and getattr(model.config, "add_time_instruction", False):
296
+ video_time, frame_time, num_frames = video_stats
297
+ time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
298
+ # 剔除原有可能存在的图片 Token 以免重叠
299
+ clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
300
+ prompt_text = f"{time_instruction}\n{clean_prompt}"
301
+
302
+ # Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN)
303
+ input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
304
+
305
+ end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
306
+ seq_len = len(end_header_seq)
307
+ pos_eh = -1
308
+ if seq_len > 0 and input_ids_text.numel() >= seq_len:
309
+ ids_list = input_ids_text.tolist()
310
+ for i in range(len(ids_list) - seq_len, -1, -1):
311
+ if ids_list[i:i + seq_len] == end_header_seq:
312
+ pos_eh = i + seq_len - 1
313
+ break
314
+ assert pos_eh != -1, " <|end_header_id|> not found"
315
+ start_pos = pos_eh + 1
316
+
317
+ labels_text = input_ids_text.clone()
318
+ labels_text[:start_pos] = -100
319
+
320
+ (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
321
+ input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
322
+ )
323
+ clean_embeds = clean_embeds.squeeze(0) # (L, D)
324
+ new_labels = new_labels.squeeze(0) # (L,)
325
+
326
+ eot_id = 126348
327
+ eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
328
+ if eot_indices.numel() > 0:
329
+ last_eot_pos = eot_indices[-1, 0]
330
+ new_labels[last_eot_pos:] = -100
331
+ asst_mask = (new_labels != -100)
332
+ asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,)
333
+
334
+ sample_cache[sid] = dict(
335
+ clean_embeds=clean_embeds,
336
+ new_labels=new_labels,
337
+ asst_idx=asst_idx,
338
+ )
339
+
340
+ lengths = []
341
+ for i in range(bsz):
342
+ sid = int(batch["sample_id"][i])
343
+ lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0]))
344
+ Lmax = max(lengths)
345
+ D = next(iter(sample_cache.values()))["clean_embeds"].shape[1]
346
+
347
+ mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
348
+
349
+ noisy_list = []
350
+ labels_list = []
351
+ p_mask_list = []
352
+ attn_list = []
353
+
354
+ for i in range(bsz):
355
+ sid = int(batch["sample_id"][i])
356
+ var_type = batch["var_type"][i]
357
+ var_param = batch["var_param"][i]
358
+ step_map = batch["step_map"][i]
359
+
360
+ clean_embeds = sample_cache[sid]["clean_embeds"].clone()
361
+ new_labels = sample_cache[sid]["new_labels"].clone()
362
+ asst_idx = sample_cache[sid]["asst_idx"]
363
+
364
+ L = clean_embeds.size(0)
365
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
366
+
367
+ if var_type == "semi_ar":
368
+ order = torch.as_tensor(step_map, device=device)
369
+ M = asst_idx.numel()
370
+ if order.numel() != M:
371
+ m = min(M, order.numel())
372
+ asst_idx = asst_idx[:m]
373
+ order = order[:m]
374
+ order_full = torch.full((L,), -1, dtype=torch.long, device=device)
375
+ order_full[asst_idx] = order
376
+ step_val = int(var_param)
377
+
378
+ p = torch.empty(L, device=device).uniform_(0.0, 1.0)
379
+ lower = 0.0 + 1e-6
380
+ upper = 1.0 - 1e-6
381
+ thresh = (upper - lower) * torch.rand(1, device=device) + lower
382
+ block_mask = (order_full == step_val) & (p < thresh)
383
+
384
+ mask_pos = (order_full > step_val) | block_mask
385
+ noisy_embeds = clean_embeds.clone()
386
+ noisy_embeds[mask_pos] = mask_embed
387
+
388
+ pmask = block_mask
389
+
390
+ elif var_type == "random_masking":
391
+ M = asst_idx.numel()
392
+ if M == 0:
393
+ noisy_embeds = clean_embeds
394
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
395
+ else:
396
+ t = float(var_param)
397
+ mask_seed = batch.get("mask_seed", [None] * bsz)[i]
398
+ if mask_seed is None:
399
+ mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
400
+ gen = torch.Generator(device="cpu")
401
+ gen.manual_seed(int(mask_seed))
402
+ rnd_cpu = torch.rand(M, generator=gen) < t
403
+ rnd = rnd_cpu.to(device=device)
404
+
405
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
406
+ pmask[asst_idx] = rnd
407
+
408
+ noisy_embeds = clean_embeds.clone()
409
+ noisy_embeds[pmask] = mask_embed
410
+
411
+ else:
412
+ continue
413
+
414
+ if not pmask.any():
415
+ if asst_idx.numel() > 0:
416
+ continue
417
+ else:
418
+ print("*********bad data*********")
419
+ print(batch["sample_id"][i])
420
+ continue
421
+
422
+ # if not pmask.any():
423
+ # print('*****no token masked*****')
424
+ # if asst_idx.numel() > 0:
425
+ # # 强制 Mask 最后一个 assistant token
426
+ # force_idx = asst_idx[-1]
427
+ # pmask[force_idx] = True
428
+ # noisy_embeds[force_idx] = mask_embed
429
+ # else:
430
+ # print("*********bad data*********")
431
+ # print(batch["sample_id"][i])
432
+ # # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过
433
+ # continue
434
+
435
+
436
+ noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
437
+ labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
438
+ p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
439
+ attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
440
+ attn[-L:] = True
441
+ attn_list.append(attn)
442
+
443
+ if len(noisy_list) == 0:
444
+ return None
445
+
446
+ noisy_batch = torch.stack(noisy_list, dim=0)
447
+ labels_batch = torch.stack(labels_list, dim=0)
448
+ p_mask_batch = torch.stack(p_mask_list, dim=0)
449
+ attn_batch = torch.stack(attn_list, dim=0)
450
+ return noisy_batch, labels_batch, p_mask_batch, attn_batch
451
+
452
+ # ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ...
453
+ def save_checkpoint(model, tokenizer, config, accelerator, name):
454
+ output_dir = Path(config.experiment.project)
455
+ output_dir.mkdir(parents=True, exist_ok=True)
456
+
457
+ checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
458
+ if accelerator.is_main_process and checkpoints_total_limit is not None:
459
+ ckpts = sorted(
460
+ [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
461
+ key=lambda p: int(p.name.split("-")[1]),
462
+ )
463
+ if len(ckpts) >= checkpoints_total_limit:
464
+ to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
465
+ logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
466
+ for p in to_remove:
467
+ shutil.rmtree(p, ignore_errors=True)
468
+
469
+ save_base = output_dir / "ckpt"
470
+ save_base.mkdir(exist_ok=True)
471
+
472
+ model_to_save = accelerator.unwrap_model(model)
473
+ state_dict = accelerator.get_state_dict(model)
474
+
475
+ if accelerator.is_main_process:
476
+ model_to_save.save_pretrained(
477
+ save_base / name,
478
+ save_function=accelerator.save,
479
+ state_dict=state_dict,
480
+ safe_serialization=True,
481
+ )
482
+ tokenizer.save_pretrained(str(save_base / name))
483
+
484
+ metadata = {
485
+ "save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
486
+ }
487
+ with (save_base / "metadata.json").open("w") as f:
488
+ json.dump(metadata, f, indent=2)
489
+
490
+ logger.info(f"Saved model + tokenizer to {save_base / name}")
491
+
492
+
493
+ def main():
494
+ config = get_config()
495
+
496
+ # TF32
497
+ if config.training.enable_tf32:
498
+ torch.backends.cuda.matmul.allow_tf32 = True
499
+ torch.backends.cudnn.benchmark = True
500
+ torch.backends.cudnn.deterministic = False
501
+
502
+ config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
503
+
504
+ # 🌟 实例化 DDP 处理器,允许存在未使用的参数
505
+ # ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
506
+ # process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600))
507
+
508
+ accelerator = Accelerator(
509
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
510
+ mixed_precision=config.training.mixed_precision,
511
+ log_with="wandb",
512
+ project_dir=config.experiment.logging_dir,
513
+ split_batches=True,
514
+ # kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去
515
+ # kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去
516
+ )
517
+
518
+ logging.basicConfig(
519
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
520
+ datefmt="%m/%d/%Y %H:%M:%S",
521
+ level=logging.INFO,
522
+ )
523
+ logger.info(accelerator.state, main_process_only=False)
524
+ if accelerator.is_local_main_process:
525
+ set_verbosity_info()
526
+ else:
527
+ set_verbosity_error()
528
+
529
+ if accelerator.is_main_process:
530
+ resume_wandb_run = config.wandb.resume
531
+ run_id = config.wandb.get("run_id", None)
532
+ if run_id is None:
533
+ resume_wandb_run = False
534
+ run_id = wandb.util.generate_id()
535
+ config.wandb.run_id = run_id
536
+ wandb_init_kwargs = dict(
537
+ name=config.experiment.project,
538
+ id=run_id,
539
+ resume=resume_wandb_run,
540
+ entity=config.wandb.get("entity", None),
541
+ config_exclude_keys=[],
542
+ )
543
+ wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
544
+ wandb_config.pop("experiment.resume_from_checkpoint", None)
545
+
546
+ accelerator.init_trackers(
547
+ config.experiment.project,
548
+ config=wandb_config,
549
+ init_kwargs={"wandb": wandb_init_kwargs},
550
+ )
551
+
552
+ if accelerator.is_main_process:
553
+ os.makedirs(config.experiment.project, exist_ok=True)
554
+ config_path = Path(config.experiment.project) / "config.yaml"
555
+ logging.info(f"Saving config to {config_path}")
556
+ OmegaConf.save(config, config_path)
557
+
558
+ if config.training.seed is not None:
559
+ set_seed(config.training.seed)
560
+
561
+ pretrained_model = config.model.pretrained_model
562
+ tokenizer, model, image_processor, _ = load_pretrained_model(
563
+ pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
564
+ )
565
+ model = model.to(accelerator.device)
566
+
567
+ special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
568
+ num_new = tokenizer.add_special_tokens(special_tokens)
569
+ if num_new > 0:
570
+ model.resize_token_embeddings(len(tokenizer))
571
+ if tokenizer.pad_token is None:
572
+ tokenizer.pad_token = tokenizer.eos_token
573
+
574
+ if getattr(config.training, "gradient_checkpointing_enable", False):
575
+ try:
576
+ model.enable_input_require_grads()
577
+ except Exception:
578
+ pass
579
+ try:
580
+ model.gradient_checkpointing_enable()
581
+ except Exception:
582
+ pass
583
+
584
+ mask_id = 126336
585
+
586
+ with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f:
587
+ data_list = json.load(f)
588
+
589
+ ds = SFTVariantDatasetLLADAV(
590
+ samples=data_list,
591
+ method=config.training.method,
592
+ lower=config.training.lower_p,
593
+ upper=config.training.upper_p,
594
+ block_size=config.training.block_size,
595
+ mask_times=config.training.mask_times_per_sample,
596
+ max_gen_length=config.training.max_gen_length
597
+ )
598
+
599
+ # Scheduler & DataLoader
600
+ total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
601
+ num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm)
602
+ num_train_epochs = config.training.num_train_epochs
603
+ max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
604
+
605
+ optimizer_config = config.optimizer.params
606
+ no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
607
+ optimizer_grouped_parameters = [
608
+ {
609
+ "params": [p for n, p in model.named_parameters() if
610
+ p.requires_grad and not any(nd in n for nd in no_decay)],
611
+ "weight_decay": optimizer_config.weight_decay,
612
+ },
613
+ {
614
+ "params": [p for n, p in model.named_parameters() if
615
+ p.requires_grad and any(nd in n for nd in no_decay)],
616
+ "weight_decay": 0.0,
617
+ },
618
+ ]
619
+ optimizer = AdamW(
620
+ optimizer_grouped_parameters,
621
+ lr=optimizer_config.learning_rate,
622
+ betas=(optimizer_config.beta1, optimizer_config.beta2),
623
+ weight_decay=optimizer_config.weight_decay,
624
+ eps=optimizer_config.epsilon,
625
+ )
626
+
627
+ lr_scheduler = get_scheduler(
628
+ config.lr_scheduler.scheduler,
629
+ optimizer=optimizer,
630
+ num_training_steps=max_train_steps,
631
+ num_warmup_steps=config.lr_scheduler.params.warmup_steps,
632
+ min_lr_scale=config.lr_scheduler.params.min_lr_scale
633
+ )
634
+
635
+ dl = DataLoader(
636
+ ds,
637
+ batch_size=config.training.batch_size_lm,
638
+ sampler=None,
639
+ collate_fn=collate_variants,
640
+ num_workers=0,
641
+ )
642
+
643
+ model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl)
644
+
645
+ logger.info("***** Running LLaDA-V SFT training *****")
646
+ logger.info(f" Num variants = {len(ds)}")
647
+ logger.info(f" Num training steps = {max_train_steps}")
648
+ logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
649
+ logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
650
+ logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
651
+
652
+ data_time_m = AverageMeter()
653
+ end = time.time()
654
+
655
+ from tqdm.auto import tqdm
656
+
657
+ def forward_process_batch(batch):
658
+ out = prepare_batch_noisy_embeds(
659
+ accelerator=accelerator,
660
+ model=model,
661
+ tokenizer=tokenizer,
662
+ image_processor=image_processor,
663
+ batch=batch,
664
+ mask_id=mask_id,
665
+ dataset_media_root=config.dataset.get("image_root", None),
666
+ )
667
+ if out is None:
668
+ return None
669
+ noisy_embeds, labels, p_mask, attn = out
670
+
671
+ outputs = model.get_model()(
672
+ inputs_embeds=noisy_embeds,
673
+ attention_mask=attn,
674
+ use_cache=False,
675
+ return_dict=True,
676
+ )
677
+ logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V)
678
+
679
+ # logits = outputs.logits.float() # (B, L, V)
680
+ log_probs = F.log_softmax(logits, dim=-1)
681
+
682
+ safe_labels = labels.clone()
683
+ safe_labels[labels == -100] = 0
684
+ logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L)
685
+
686
+ loss_tok = - (logp_tok * p_mask).sum(dim=1)
687
+ num_mask = torch.clamp(p_mask.sum(dim=1), min=1)
688
+ loss = (loss_tok / num_mask).mean()
689
+ return loss
690
+
691
+ first_epoch = 0
692
+ grad_accum = config.training.gradient_accumulation_steps
693
+
694
+ for epoch in range(first_epoch, num_train_epochs):
695
+ model.train()
696
+ progress_bar = tqdm(
697
+ dl,
698
+ desc=f"Epoch {epoch+1}/{num_train_epochs}",
699
+ disable=not accelerator.is_local_main_process,
700
+ dynamic_ncols=True,
701
+ leave=True
702
+ )
703
+
704
+ # for step, batch in enumerate(progress_bar, start=1):
705
+ # data_time_m.update(time.time() - end)
706
+
707
+ # # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器
708
+ # with accelerator.accumulate(model):
709
+ # # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理
710
+ # loss = forward_process_batch(batch)
711
+
712
+ # accelerator.backward(loss)
713
+
714
+ # # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪
715
+ # if accelerator.sync_gradients:
716
+ # if config.training.max_grad_norm is not None:
717
+ # accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
718
+
719
+ # optimizer.step()
720
+ # lr_scheduler.step()
721
+ # optimizer.zero_grad(set_to_none=True)
722
+
723
+ # # 警告:坚决删掉这里的 torch.cuda.empty_cache()
724
+
725
+ step = 0
726
+ for step, batch in enumerate(progress_bar, start=1):
727
+ data_time_m.update(time.time() - end)
728
+
729
+ loss = forward_process_batch(batch)
730
+ if loss is None:
731
+ continue
732
+ loss = loss / grad_accum
733
+ accelerator.backward(loss)
734
+
735
+ if step % grad_accum == 0:
736
+ if config.training.max_grad_norm is not None:
737
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
738
+ optimizer.step()
739
+ lr_scheduler.step()
740
+ optimizer.zero_grad(set_to_none=True)
741
+ torch.cuda.empty_cache()
742
+
743
+ if step % grad_accum != 0:
744
+ if config.training.max_grad_norm is not None:
745
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
746
+ optimizer.step()
747
+ lr_scheduler.step()
748
+ optimizer.zero_grad(set_to_none=True)
749
+ torch.cuda.empty_cache()
750
+
751
+ accelerator.wait_for_everyone()
752
+ save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
753
+ accelerator.end_training()
754
+
755
+
756
+ if __name__ == "__main__":
757
+ main()
wandb/run-20260322_031609-y30zeekk/files/diff.patch ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..5701c74 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,62 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [],
158
+ + "source": [
159
+ + "import json\n",
160
+ + "\n",
161
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
162
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
163
+ + "save_data_list = []\n",
164
+ + "save_num = 0\n",
165
+ + "with open(data_path, 'r') as fin, \\\n",
166
+ + " open(save_path, 'w') as fout:\n",
167
+ + " raw_data = json.load(fin)\n",
168
+ + " for data in raw_data:\n",
169
+ + " if save_num < 100:\n",
170
+ + " save_data_list.append(data)\n",
171
+ + " save_num += 1\n",
172
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
173
+ + " print(len(save_data_list))\n"
174
+ + ]
175
+ }
176
+ ],
177
+ "metadata": {
178
+ @@ -85,7 +141,7 @@
179
+ "name": "python",
180
+ "nbconvert_exporter": "python",
181
+ "pygments_lexer": "ipython3",
182
+ - "version": "3.10.18"
183
+ + "version": "3.10.19"
184
+ }
185
+ },
186
+ "nbformat": 4,
187
+ diff --git a/rl_v.py b/rl_v.py
188
+ index 8f2c0f1..79d2e5b 100644
189
+ --- a/rl_v.py
190
+ +++ b/rl_v.py
191
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
192
+ def env_prefix(env) -> str:
193
+ return (
194
+ "source ~/.bashrc && "
195
+ - f"source activate {env} && "
196
+ + "source /data8/runqing/miniconda/bin/activate && "
197
+ + "conda init && "
198
+ + f"conda activate {env} && "
199
+ )
200
+
201
+ config = get_config()
202
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
203
+ index 4068fc8..d284a23 100644
204
+ --- a/sample/llava/conversation.py
205
+ +++ b/sample/llava/conversation.py
206
+ @@ -99,7 +99,7 @@ class Conversation:
207
+ if self.version == "llama_v3":
208
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
209
+ elif self.version == "llava_llada":
210
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
211
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
212
+ else:
213
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
214
+ chat_template_messages = [{"role": "system", "content": self.system}]
215
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
216
+ index 3db5302..96e33b5 100644
217
+ --- a/sample/llava/model/builder.py
218
+ +++ b/sample/llava/model/builder.py
219
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
220
+ elif "llada" in model_name.lower():
221
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
222
+
223
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
224
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
225
+ +
226
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
227
+ if customized_config is None:
228
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
229
+ else:
230
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
231
+ index 4068fc8..d284a23 100644
232
+ --- a/train/llava/conversation.py
233
+ +++ b/train/llava/conversation.py
234
+ @@ -99,7 +99,7 @@ class Conversation:
235
+ if self.version == "llama_v3":
236
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
237
+ elif self.version == "llava_llada":
238
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
239
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
240
+ else:
241
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
242
+ chat_template_messages = [{"role": "system", "content": self.system}]
243
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
244
+ index 3db5302..aafab95 100644
245
+ --- a/train/llava/model/builder.py
246
+ +++ b/train/llava/model/builder.py
247
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
248
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
249
+ elif "llada" in model_name.lower():
250
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
251
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
252
+
253
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
254
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
255
+ if customized_config is None:
256
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
257
+ else:
258
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
259
+ index 3da28dc..8b17da3 100644
260
+ --- a/train/llava/model/language_model/llava_llada.py
261
+ +++ b/train/llava/model/language_model/llava_llada.py
262
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
263
+ dpo_forward: Optional[bool] = None,
264
+ cache_position=None,
265
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
266
+ -
267
+ + conversation_ids = None
268
+ if inputs_embeds is None and attention_mask is not None:
269
+ # donate multi-dialogue
270
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..5701c74 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,62 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [],
158
+ + "source": [
159
+ + "import json\n",
160
+ + "\n",
161
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
162
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
163
+ + "save_data_list = []\n",
164
+ + "save_num = 0\n",
165
+ + "with open(data_path, 'r') as fin, \\\n",
166
+ + " open(save_path, 'w') as fout:\n",
167
+ + " raw_data = json.load(fin)\n",
168
+ + " for data in raw_data:\n",
169
+ + " if save_num < 100:\n",
170
+ + " save_data_list.append(data)\n",
171
+ + " save_num += 1\n",
172
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
173
+ + " print(len(save_data_list))\n"
174
+ + ]
175
+ }
176
+ ],
177
+ "metadata": {
178
+ @@ -85,7 +141,7 @@
179
+ "name": "python",
180
+ "nbconvert_exporter": "python",
181
+ "pygments_lexer": "ipython3",
182
+ - "version": "3.10.18"
183
+ + "version": "3.10.19"
184
+ }
185
+ },
186
+ "nbformat": 4,
187
+ diff --git a/rl_v.py b/rl_v.py
188
+ index 8f2c0f1..79d2e5b 100644
189
+ --- a/rl_v.py
190
+ +++ b/rl_v.py
191
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
192
+ def env_prefix(env) -> str:
193
+ return (
194
+ "source ~/.bashrc && "
195
+ - f"source activate {env} && "
196
+ + "source /data8/runqing/miniconda/bin/activate && "
197
+ + "conda init && "
198
+ + f"conda activate {env} && "
199
+ )
200
+
201
+ config = get_config()
202
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
203
+ index 4068fc8..d284a23 100644
204
+ --- a/sample/llava/conversation.py
205
+ +++ b/sample/llava/conversation.py
206
+ @@ -99,7 +99,7 @@ class Conversation:
207
+ if self.version == "llama_v3":
208
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
209
+ elif self.version == "llava_llada":
210
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
211
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
212
+ else:
213
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
214
+ chat_template_messages = [{"role": "system", "content": self.system}]
215
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
216
+ index 3db5302..96e33b5 100644
217
+ --- a/sample/llava/model/builder.py
218
+ +++ b/sample/llava/model/builder.py
219
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
220
+ elif "llada" in model_name.lower():
221
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
222
+
223
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
224
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
225
+ +
226
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
227
+ if customized_config is None:
228
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
229
+ else:
230
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
231
+ index 4068fc8..d284a23 100644
232
+ --- a/train/llava/conversation.py
233
+ +++ b/train/llava/conversation.py
234
+ @@ -99,7 +99,7 @@ class Conversation:
235
+ if self.version == "llama_v3":
236
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
237
+ elif self.version == "llava_llada":
238
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
239
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
240
+ else:
241
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
242
+ chat_template_messages = [{"role": "system", "content": self.system}]
243
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
244
+ index 3db5302..aafab95 100644
245
+ --- a/train/llava/model/builder.py
246
+ +++ b/train/llava/model/builder.py
247
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
248
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
249
+ elif "llada" in model_name.lower():
250
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
251
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
252
+
253
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
254
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
255
+ if customized_config is None:
256
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
257
+ else:
258
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
259
+ index 3da28dc..8b17da3 100644
260
+ --- a/train/llava/model/language_model/llava_llada.py
261
+ +++ b/train/llava/model/language_model/llava_llada.py
262
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
263
+ dpo_forward: Optional[bool] = None,
264
+ cache_position=None,
265
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
266
+ -
267
+ + conversation_ids = None
268
+ if inputs_embeds is None and attention_mask is not None:
269
+ # donate multi-dialogue
270
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
wandb/run-20260322_031609-y30zeekk/files/output.log ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 03/22/2026 03:16:15 - INFO - root - Saving config to sft_lladav_video_reversal_3_21/config.yaml
2
+ Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/LLaDA-V
3
+ Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
4
+ Loading checkpoint shards: 100%|██████████| 6/6 [00:25<00:00, 4.19s/it]
5
+ Rank 0: Model Class: LlavaLLaDAModelLM
6
+ /data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
7
+ If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
8
+ warnings.warn(
9
+ Before initializing optimizer states
10
+ MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB
11
+ CPU Virtual Memory: used = 133.3 GB, percent = 6.6%
12
+ After initializing optimizer states
13
+ MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB
14
+ CPU Virtual Memory: used = 158.85 GB, percent = 7.9%
15
+ After initializing ZeRO optimizer
16
+ MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB
17
+ CPU Virtual Memory: used = 160.29 GB, percent = 8.0%
18
+ 03/22/2026 03:18:02 - INFO - __main__ - ***** Running LLaDA-V SFT training *****
19
+ 03/22/2026 03:18:02 - INFO - __main__ - Num variants = 39024
20
+ 03/22/2026 03:18:02 - INFO - __main__ - Num training steps = 611
21
+ 03/22/2026 03:18:02 - INFO - __main__ - Instantaneous batch size per device = 2
22
+ 03/22/2026 03:18:02 - INFO - __main__ - Total train batch size (parallel * accum) = 64
23
+ 03/22/2026 03:18:02 - INFO - __main__ - Gradient Accumulation steps = 4
24
+ Epoch 1/1: 0%| | 0/2439 [00:00<?, ?it/s]/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/checkpoint.py:87: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
25
+ warnings.warn(
26
+ Epoch 1/1: 2%|▏ | 39/2439 [05:16<7:38:40, 11.47s/it]
wandb/run-20260322_031609-y30zeekk/files/requirements.txt ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ transformers==4.52.4
2
+ nvidia-nvtx-cu12==12.4.127
3
+ deepspeed==0.18.6
4
+ typing-inspection==0.4.2
5
+ scipy==1.15.3
6
+ MarkupSafe==3.0.3
7
+ nvidia-cuda-runtime-cu12==12.4.127
8
+ yarl==1.22.0
9
+ tornado==6.5.4
10
+ nvidia-curand-cu12==10.3.5.147
11
+ nest-asyncio==1.6.0
12
+ safetensors==0.7.0
13
+ async-timeout==5.0.1
14
+ typing_extensions==4.15.0
15
+ asttokens==3.0.1
16
+ cuda-python==13.1.1
17
+ executing==2.2.1
18
+ nvidia-cudnn-frontend==1.18.0
19
+ comm==0.2.3
20
+ tokenizers==0.21.4
21
+ six==1.17.0
22
+ Pygments==2.19.2
23
+ nvidia-cusparselt-cu12==0.6.2
24
+ apache-tvm-ffi==0.1.8.post2
25
+ flash_attn==2.7.4.post1
26
+ tqdm==4.67.3
27
+ tensorboard-data-server==0.7.2
28
+ annotated-types==0.7.0
29
+ math-verify==0.9.0
30
+ idna==3.11
31
+ regex==2026.1.15
32
+ nvidia-cutlass-dsl-libs-base==4.4.0
33
+ flashinfer-python==0.6.3
34
+ click==8.3.1
35
+ pytz==2025.2
36
+ wheel==0.46.3
37
+ ipykernel==7.2.0
38
+ stack-data==0.6.3
39
+ absl-py==2.4.0
40
+ wandb==0.25.0
41
+ pandas==2.3.3
42
+ termcolor==3.3.0
43
+ sympy==1.13.1
44
+ joblib==1.5.3
45
+ prompt_toolkit==3.0.52
46
+ attrs==25.4.0
47
+ nvidia-cusolver-cu12==11.6.1.9
48
+ xxhash==3.6.0
49
+ aiohappyeyeballs==2.6.1
50
+ python-dateutil==2.9.0.post0
51
+ frozenlist==1.8.0
52
+ nvidia-ml-py==13.590.48
53
+ multiprocess==0.70.18
54
+ decorator==5.2.1
55
+ cuda-pathfinder==1.3.4
56
+ multidict==6.7.1
57
+ pure_eval==0.2.3
58
+ nvidia-cufft-cu12==11.2.1.3
59
+ omegaconf==2.3.0
60
+ PyYAML==6.0.3
61
+ debugpy==1.8.20
62
+ psutil==7.2.2
63
+ ninja==1.13.0
64
+ nvidia-cublas-cu12==12.4.5.8
65
+ triton==3.2.0
66
+ antlr4-python3-runtime==4.9.3
67
+ peft==0.18.1
68
+ aiosignal==1.4.0
69
+ accelerate==1.12.0
70
+ liger_kernel==0.7.0
71
+ anyio==4.12.1
72
+ decord==0.6.0
73
+ h11==0.16.0
74
+ urllib3==2.6.3
75
+ wcwidth==0.6.0
76
+ networkx==3.4.2
77
+ huggingface_hub==0.36.2
78
+ tzdata==2025.3
79
+ xformers==0.0.29.post3
80
+ tabulate==0.9.0
81
+ certifi==2026.1.4
82
+ tensorboard==2.20.0
83
+ matplotlib-inline==0.2.1
84
+ pip==26.0.1
85
+ smmap==5.0.2
86
+ nvidia-cuda-cupti-cu12==12.4.127
87
+ Markdown==3.10.2
88
+ nvidia-nccl-cu12==2.21.5
89
+ aiohttp==3.13.3
90
+ scikit-learn==1.7.2
91
+ nvidia-cutlass-dsl==4.4.0
92
+ setuptools==82.0.0
93
+ jupyter_core==5.9.1
94
+ asyncio==4.0.0
95
+ einops==0.8.2
96
+ pydantic==2.12.5
97
+ av==16.1.0
98
+ protobuf==6.33.5
99
+ evaluate==0.4.6
100
+ Jinja2==3.1.6
101
+ exceptiongroup==1.3.1
102
+ ipython==8.38.0
103
+ sentry-sdk==2.53.0
104
+ httpx==0.28.1
105
+ msgpack==1.1.2
106
+ platformdirs==4.9.2
107
+ hjson==3.1.0
108
+ torchvision==0.21.0
109
+ parso==0.8.6
110
+ cuda-bindings==13.1.1
111
+ pexpect==4.9.0
112
+ threadpoolctl==3.6.0
113
+ pyarrow==23.0.1
114
+ jedi==0.19.2
115
+ nvidia-cuda-nvrtc-cu12==12.4.127
116
+ Werkzeug==3.1.5
117
+ fsspec==2025.10.0
118
+ pydantic_core==2.41.5
119
+ GitPython==3.1.46
120
+ propcache==0.4.1
121
+ mpmath==1.3.0
122
+ nvidia-nvjitlink-cu12==12.4.127
123
+ nvidia-cusparse-cu12==12.3.1.170
124
+ hf-xet==1.2.0
125
+ charset-normalizer==3.4.4
126
+ dill==0.4.0
127
+ datasets==4.5.0
128
+ py-cpuinfo==9.0.0
129
+ sentencepiece==0.2.1
130
+ latex2sympy2_extended==1.11.0
131
+ jupyter_client==8.8.0
132
+ pyzmq==27.1.0
133
+ numpy==2.2.6
134
+ gitdb==4.0.12
135
+ grpcio==1.78.0
136
+ packaging==26.0
137
+ requests==2.32.5
138
+ nvidia-cudnn-cu12==9.1.0.70
139
+ httpcore==1.0.9
140
+ traitlets==5.14.3
141
+ filelock==3.24.2
142
+ ptyprocess==0.7.0
143
+ pillow==12.1.1
144
+ torch==2.6.0
145
+ more-itertools==10.8.0
146
+ wheel==0.46.3
147
+ jaraco.text==4.0.0
148
+ zipp==3.23.0
149
+ autocommand==2.2.2
150
+ jaraco.context==6.1.0
151
+ jaraco.functools==4.4.0
152
+ tomli==2.4.0
153
+ backports.tarfile==1.2.0
154
+ importlib_metadata==8.7.1
155
+ platformdirs==4.4.0
156
+ packaging==26.0
wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.19",
4
+ "startedAt": "2026-03-21T19:16:09.677405Z",
5
+ "args": [
6
+ "config=configs/sft_lladav_video_reversal.yaml"
7
+ ],
8
+ "program": "/data4/runqing/dLLM-RL/train/sft_lladav_video_reversal.py",
9
+ "codePath": "train/sft_lladav_video_reversal.py",
10
+ "codePathLocal": "train/sft_lladav_video_reversal.py",
11
+ "git": {
12
+ "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
13
+ "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
14
+ },
15
+ "email": "shixi_runqing@staff.weibo.com",
16
+ "root": "/data4/runqing/dLLM-RL",
17
+ "host": "b1301b38c75f",
18
+ "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
19
+ "cpu_count": 96,
20
+ "cpu_count_logical": 192,
21
+ "gpu": "NVIDIA H800",
22
+ "gpu_count": 8,
23
+ "disk": {
24
+ "/": {
25
+ "total": "806578724864",
26
+ "used": "571700989952"
27
+ }
28
+ },
29
+ "memory": {
30
+ "total": "2164113813504"
31
+ },
32
+ "gpu_nvidia": [
33
+ {
34
+ "name": "NVIDIA H800",
35
+ "memoryTotal": "85520809984",
36
+ "cudaCores": 16896,
37
+ "architecture": "Hopper",
38
+ "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
39
+ },
40
+ {
41
+ "name": "NVIDIA H800",
42
+ "memoryTotal": "85520809984",
43
+ "cudaCores": 16896,
44
+ "architecture": "Hopper",
45
+ "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
46
+ },
47
+ {
48
+ "name": "NVIDIA H800",
49
+ "memoryTotal": "85520809984",
50
+ "cudaCores": 16896,
51
+ "architecture": "Hopper",
52
+ "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
53
+ },
54
+ {
55
+ "name": "NVIDIA H800",
56
+ "memoryTotal": "85520809984",
57
+ "cudaCores": 16896,
58
+ "architecture": "Hopper",
59
+ "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
60
+ },
61
+ {
62
+ "name": "NVIDIA H800",
63
+ "memoryTotal": "85520809984",
64
+ "cudaCores": 16896,
65
+ "architecture": "Hopper",
66
+ "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
67
+ },
68
+ {
69
+ "name": "NVIDIA H800",
70
+ "memoryTotal": "85520809984",
71
+ "cudaCores": 16896,
72
+ "architecture": "Hopper",
73
+ "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
74
+ },
75
+ {
76
+ "name": "NVIDIA H800",
77
+ "memoryTotal": "85520809984",
78
+ "cudaCores": 16896,
79
+ "architecture": "Hopper",
80
+ "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
81
+ },
82
+ {
83
+ "name": "NVIDIA H800",
84
+ "memoryTotal": "85520809984",
85
+ "cudaCores": 16896,
86
+ "architecture": "Hopper",
87
+ "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
88
+ }
89
+ ],
90
+ "cudaVersion": "12.4",
91
+ "writerId": "0nbl2szyao6zltkyl1iimb0wrhiws85w"
92
+ }
wandb/run-20260322_031609-y30zeekk/logs/debug-core.log ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-22T03:16:09.743285688+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp2a9va2fu/port-917982.txt","pid":917982,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-03-22T03:16:09.744267653+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":917982}
3
+ {"time":"2026-03-22T03:16:09.744257174+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-917982-918484-3562169965/socket","Net":"unix"}}
4
+ {"time":"2026-03-22T03:16:09.925032752+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-03-22T03:16:09.931813709+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"y30zeekk","id":"1(@)"}
6
+ {"time":"2026-03-22T03:16:14.939482566+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"y30zeekk","id":"1(@)"}
7
+ {"time":"2026-03-22T03:16:20.158587842+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"txikwagb0eg1"}
wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-22T03:16:09.932043851+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
2
+ {"time":"2026-03-22T03:16:14.939035914+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
3
+ {"time":"2026-03-22T03:16:14.939210648+08:00","level":"INFO","msg":"stream: created new stream","id":"y30zeekk"}
4
+ {"time":"2026-03-22T03:16:14.939296134+08:00","level":"INFO","msg":"handler: started","stream_id":"y30zeekk"}
5
+ {"time":"2026-03-22T03:16:14.939467726+08:00","level":"INFO","msg":"stream: started","id":"y30zeekk"}
6
+ {"time":"2026-03-22T03:16:14.939512912+08:00","level":"INFO","msg":"sender: started","stream_id":"y30zeekk"}
7
+ {"time":"2026-03-22T03:16:14.939515245+08:00","level":"INFO","msg":"writer: started","stream_id":"y30zeekk"}
8
+ {"time":"2026-03-22T03:16:14.942265397+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
9
+ {"time":"2026-03-22T03:16:20.122143616+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=543a564ab27a29e0506e5496b121bfa5ec3df952fb270dc6cbc8e792933f1a0d"}
10
+ {"time":"2026-03-22T03:16:20.122295801+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch</Resource><RequestId>189EF14496B0C686</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=543a564ab27a29e0506e5496b121bfa5ec3df952fb270dc6cbc8e792933f1a0d, Size: 11427}"}
11
+ {"time":"2026-03-22T03:16:20.123748872+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=65fe9536dca78a54fcb1be38cc2071ac77844ac78ae053d4f2dda5671108cda7"}
12
+ {"time":"2026-03-22T03:16:20.123818956+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt</Resource><RequestId>189EF14496BBA833</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=65fe9536dca78a54fcb1be38cc2071ac77844ac78ae053d4f2dda5671108cda7, Size: 2966}"}
13
+ {"time":"2026-03-22T03:16:20.124069975+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=76db83e542423b361e9411c108e0835fa100fe9691efcbc1f98e5e3a28fedecf"}
14
+ {"time":"2026-03-22T03:16:20.124180799+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json</Resource><RequestId>189EF14496AFDED4</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=76db83e542423b361e9411c108e0835fa100fe9691efcbc1f98e5e3a28fedecf, Size: 2650}"}
15
+ {"time":"2026-03-22T03:16:20.126422985+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8f5ee1f91644676947fa6db83d772470d8ed9fdcf739aa7e905ac62cb16f53d"}
16
+ {"time":"2026-03-22T03:16:20.126566466+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189EF14496C34324</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8f5ee1f91644676947fa6db83d772470d8ed9fdcf739aa7e905ac62cb16f53d, Size: 11427}"}
17
+ {"time":"2026-03-22T03:16:20.13421156+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59e9f74c84de8a143e6c6b9f2aba14af539692b331769d7512155279863bd62c"}
18
+ {"time":"2026-03-22T03:16:20.134366382+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py</Resource><RequestId>189EF14496CC1BA5</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py, Name: code/train/sft_lladav_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59e9f74c84de8a143e6c6b9f2aba14af539692b331769d7512155279863bd62c, Size: 29865}"}
wandb/run-20260322_031609-y30zeekk/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
2
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Configure stats pid to 917982
3
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/logs/debug.log
5
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log
6
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():844] calling init triggers
7
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
+ config: {'_wandb': {'code_path': 'code/train/sft_lladav_video_reversal.py'}}
9
+ 2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():892] starting backend
10
+ 2026-03-22 03:16:09,925 INFO MainThread:917982 [wandb_init.py:init():895] sending inform_init request
11
+ 2026-03-22 03:16:09,928 INFO MainThread:917982 [wandb_init.py:init():903] backend started and connected
12
+ 2026-03-22 03:16:09,929 INFO MainThread:917982 [wandb_init.py:init():973] updated telemetry
13
+ 2026-03-22 03:16:09,935 INFO MainThread:917982 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
+ 2026-03-22 03:16:15,046 INFO MainThread:917982 [wandb_init.py:init():1042] starting run threads in backend
15
+ 2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_console_start():2524] atexit reg
16
+ 2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-03-22 03:16:15,157 INFO MainThread:917982 [wandb_init.py:init():1082] run started, returning control to user process
20
+ 2026-03-22 03:16:15,159 INFO MainThread:917982 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'y30zeekk', 'experiment.project': 'sft_lladav_video_reversal_3_21', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lladav_video_reversal_3_21/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train_cleaned', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 4, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.1, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lladav_video_reversal.yaml'}
wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb ADDED
Binary file (98.3 kB). View file
 
wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py ADDED
@@ -0,0 +1,806 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
4
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
5
+ from datetime import timedelta
6
+ import json
7
+ import logging
8
+ import math
9
+ import shutil
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Dict, List, Any
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from torch.optim import AdamW
18
+ from torch.utils.data import Dataset, DataLoader
19
+
20
+ from omegaconf import OmegaConf
21
+ import wandb
22
+
23
+ from accelerate import Accelerator
24
+ from accelerate.logging import get_logger
25
+ from accelerate.utils import set_seed
26
+
27
+ from train.utils import get_config, flatten_omega_conf, AverageMeter
28
+ from models.lr_schedulers import get_scheduler
29
+ from models.logging import set_verbosity_info, set_verbosity_error
30
+
31
+ from transformers.utils import logging as hf_logging
32
+ hf_logging.set_verbosity_error()
33
+
34
+ from llava.model.builder import load_pretrained_model
35
+ from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord
36
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
37
+ from llava.conversation import conv_templates
38
+
39
+ from termcolor import cprint
40
+
41
+ logger = get_logger(__name__, log_level="INFO")
42
+
43
+
44
+ def collapse_k_unique(lst, k: int):
45
+ if k <= 0:
46
+ raise ValueError("k must be > 0")
47
+ uniq = sorted(set(lst))
48
+ mapping = {}
49
+ n = len(uniq)
50
+ for idx, val in enumerate(uniq):
51
+ group = idx // k
52
+ end_idx = min((group + 1) * k - 1, n - 1)
53
+ rep = uniq[end_idx]
54
+ mapping[val] = rep
55
+ return [mapping[x] for x in lst]
56
+
57
+
58
+ def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
59
+ import copy
60
+ conv = copy.deepcopy(conv_templates["llava_llada"])
61
+ conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
62
+ conv.append_message(conv.roles[1], response_text)
63
+ return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
64
+
65
+
66
+ def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
67
+ conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
68
+ ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
69
+ return ids.squeeze(0)
70
+
71
+
72
+ def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
73
+ from PIL import Image
74
+ from types import SimpleNamespace
75
+
76
+ if isinstance(model_cfg, dict):
77
+ model_cfg = SimpleNamespace(**model_cfg)
78
+
79
+ if media_type == "image":
80
+ image = Image.open(abs_path).convert("RGB")
81
+ if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
82
+ setattr(model_cfg, "image_aspect_ratio", "pad")
83
+ if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
84
+ setattr(model_cfg, "mm_patch_merge_type", "flat")
85
+
86
+ image_tensor_list = process_images([image], image_processor, model_cfg)
87
+ image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
88
+ image_sizes = [image.size]
89
+ return image_tensor_list, image_sizes, None
90
+
91
+ elif media_type == "video":
92
+ if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
93
+ if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 0)
94
+ if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", False)
95
+
96
+ video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
97
+ image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
98
+ image_tensor = image_tensor.to(dtype=dtype, device=device)
99
+
100
+ if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
101
+ v_size = (video_frames.shape[2], video_frames.shape[1])
102
+ else:
103
+ v_size = (0, 0)
104
+
105
+ return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
106
+
107
+
108
+ def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
109
+ out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
110
+ out[-x.numel():] = x
111
+ return out
112
+
113
+
114
+ def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
115
+ if x.size(0) == L:
116
+ return x
117
+ out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
118
+ out[-x.size(0):] = x
119
+ return out
120
+
121
+
122
+ def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
123
+ p = sample.get(f"{media_type}_abs_path")
124
+ if p is None:
125
+ p = sample.get(media_type) or sample.get(f"{media_type}_path")
126
+ if p is None:
127
+ return None
128
+ if not os.path.isabs(p) and dataset_root:
129
+ p = os.path.join(dataset_root, p)
130
+ try:
131
+ return os.path.abspath(p)
132
+ except Exception:
133
+ return p
134
+
135
+
136
+ class RLVariantDatasetLLADAV(Dataset):
137
+ def __init__(self, samples: List[dict], config):
138
+ super().__init__()
139
+ self.samples = samples
140
+ self.config = config
141
+ self.variants: List[dict] = []
142
+
143
+ # logp_old_tok mapping: idx -> tensor
144
+ self.logp_old_tok: Dict[int, torch.Tensor] = {}
145
+
146
+ self._build_variants()
147
+
148
+ def _build_variants(self):
149
+ sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
150
+ question_prompt = "\nOnly give the best option."
151
+ multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
152
+
153
+ rng = np.random.default_rng(seed=self.config.training.seed or 42)
154
+ idx_counter = 0
155
+
156
+ for sid, s in enumerate(self.samples):
157
+ # Rollout produces "extracted_output" or "response", fall back if needed
158
+ response_text = s.get("response", s.get("extracted_output", ""))
159
+ if isinstance(response_text, list) and len(response_text) > 0:
160
+ response_text = response_text[0] # Just in case it's nested
161
+
162
+ # If options exist (video multiple choice), construct full prompt
163
+ if "options" in s:
164
+ prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt
165
+ else:
166
+ prompt_text = s["prompt"]
167
+
168
+ step_map = s.get("step_map", [])
169
+
170
+ if self.config.training.method == "TraceRL":
171
+ order_list = collapse_k_unique(step_map, self.config.training.shrink)
172
+ uniq_steps = sorted(set(order_list))
173
+ for st in uniq_steps:
174
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None))
175
+ idx_counter += 1
176
+
177
+ elif self.config.training.method == "random_masking":
178
+ for _ in range(self.config.training.mask_times_per_sample):
179
+ t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
180
+ mask_seed = int(rng.integers(0, 2**31 - 1))
181
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed))
182
+ idx_counter += 1
183
+
184
+ elif self.config.training.method == "coupled":
185
+ for _ in range(self.config.training.mask_times_per_sample):
186
+ t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
187
+ mask_seed = int(rng.integers(0, 2**31 - 1))
188
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed))
189
+ idx_counter += 1
190
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed))
191
+ idx_counter += 1
192
+ else:
193
+ raise ValueError(f"Unknown RL method: {self.config.training.method}")
194
+
195
+ def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed):
196
+ return dict(
197
+ idx=idx,
198
+ sample_id=sid,
199
+ prompt=prompt,
200
+ response=response,
201
+ image_abs_path=s.get("image_abs_path", None),
202
+ video_abs_path=s.get("video_abs_path", None),
203
+ reward=s.get("reward", 0.0),
204
+ var_type=var_type,
205
+ var_param=var_param,
206
+ step_map=step_map,
207
+ mask_seed=mask_seed,
208
+ )
209
+
210
+ def __len__(self):
211
+ return len(self.variants)
212
+
213
+ def __getitem__(self, idx):
214
+ return self.variants[idx]
215
+
216
+
217
+ def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
218
+ out: Dict[str, List[Any]] = {}
219
+ keys = batch[0].keys()
220
+ for k in keys:
221
+ out[k] = [b[k] for b in batch]
222
+ return out
223
+
224
+
225
+ @torch.no_grad()
226
+ def prepare_batch_noisy_embeds_rl(
227
+ accelerator,
228
+ model,
229
+ tokenizer,
230
+ image_processor,
231
+ batch: Dict[str, List[Any]],
232
+ mask_id: int,
233
+ dataset_media_root: str | None,
234
+ ) -> tuple | None:
235
+
236
+ device = accelerator.device
237
+ bsz = len(batch["idx"])
238
+
239
+ group: Dict[int, List[int]] = {}
240
+ for i, sid in enumerate(batch["sample_id"]):
241
+ group.setdefault(int(sid), []).append(i)
242
+
243
+ sample_cache: Dict[int, Dict[str, Any]] = {}
244
+ for sid, pos_list in group.items():
245
+ first_pos = pos_list[0]
246
+ prompt_text = batch["prompt"][first_pos]
247
+ response_text = batch["response"][first_pos]
248
+
249
+ sample_stub = {
250
+ "image_abs_path": batch.get("image_abs_path", [None])[first_pos],
251
+ "video_abs_path": batch.get("video_abs_path", [None])[first_pos],
252
+ }
253
+
254
+ media_type = None
255
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
256
+ if p_abs is not None:
257
+ media_type = "video"
258
+ else:
259
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
260
+ if p_abs is not None:
261
+ media_type = "image"
262
+
263
+ img_list, image_sizes, modalities = None, None, []
264
+ if p_abs is not None:
265
+ img_list, image_sizes, video_stats = load_and_process_media(
266
+ p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
267
+ )
268
+ modalities = [media_type]
269
+
270
+ if media_type == "video" and getattr(model.config, "add_time_instruction", False):
271
+ video_time, frame_time, num_frames = video_stats
272
+ time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
273
+ clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
274
+ prompt_text = f"{time_instruction}\n{clean_prompt}"
275
+
276
+ input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
277
+ end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
278
+ seq_len = len(end_header_seq)
279
+ pos_eh = -1
280
+ if seq_len > 0 and input_ids_text.numel() >= seq_len:
281
+ ids_list = input_ids_text.tolist()
282
+ for i in range(len(ids_list) - seq_len, -1, -1):
283
+ if ids_list[i:i + seq_len] == end_header_seq:
284
+ pos_eh = i + seq_len - 1
285
+ break
286
+ assert pos_eh != -1, " <|end_header_id|> not found"
287
+ start_pos = pos_eh + 1
288
+
289
+ labels_text = input_ids_text.clone()
290
+ labels_text[:start_pos] = -100
291
+
292
+ (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
293
+ input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
294
+ )
295
+ clean_embeds = clean_embeds.squeeze(0)
296
+ new_labels = new_labels.squeeze(0)
297
+
298
+ eot_id = 126348
299
+ eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
300
+ if eot_indices.numel() > 0:
301
+ last_eot_pos = eot_indices[-1, 0]
302
+ new_labels[last_eot_pos:] = -100
303
+ asst_mask = (new_labels != -100)
304
+ asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1)
305
+
306
+ sample_cache[sid] = dict(
307
+ clean_embeds=clean_embeds,
308
+ new_labels=new_labels,
309
+ asst_idx=asst_idx,
310
+ )
311
+
312
+ lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)]
313
+ Lmax = max(lengths)
314
+ mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
315
+
316
+ noisy_list, labels_list, p_mask_list, attn_list = [], [], [], []
317
+ reward_list, valid_idxs = [], []
318
+
319
+ for i in range(bsz):
320
+ sid = int(batch["sample_id"][i])
321
+ var_type = batch["var_type"][i]
322
+ var_param = batch["var_param"][i]
323
+
324
+ clean_embeds = sample_cache[sid]["clean_embeds"].clone()
325
+ new_labels = sample_cache[sid]["new_labels"].clone()
326
+ asst_idx = sample_cache[sid]["asst_idx"]
327
+
328
+ L = clean_embeds.size(0)
329
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
330
+ M = asst_idx.numel()
331
+
332
+ if M == 0:
333
+ continue # skip badly formatted inputs
334
+
335
+ if var_type == "TraceRL":
336
+ step_map = batch["step_map"][i]
337
+ order = torch.as_tensor(step_map, device=device)
338
+ m_len = min(M, order.numel())
339
+ order_full = torch.full((L,), -1, dtype=torch.long, device=device)
340
+ order_full[asst_idx[:m_len]] = order[:m_len]
341
+ step_val = int(var_param)
342
+
343
+ tgt_mask = (order_full == step_val)
344
+ pmask = tgt_mask
345
+ mask_pos = (order_full >= step_val)
346
+
347
+ noisy_embeds = clean_embeds.clone()
348
+ noisy_embeds[mask_pos] = mask_embed
349
+
350
+ elif var_type == "random_masking" or var_type.startswith("coupled"):
351
+ if M == 0:
352
+ # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward
353
+ noisy_embeds = clean_embeds.clone()
354
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
355
+ else:
356
+ t = float(var_param)
357
+ # 更安全的 seed 获取方式
358
+ mask_seed = batch.get("mask_seed", [None] * bsz)[i]
359
+ if mask_seed is None:
360
+ mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
361
+
362
+ gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
363
+ rnd_cpu = torch.rand(M, generator=gen) < t
364
+
365
+ # 如果是 coupled_comp (对比样本),需要取反
366
+ if var_type == "coupled_comp":
367
+ rnd_cpu = ~rnd_cpu
368
+
369
+ rnd = rnd_cpu.to(device)
370
+
371
+ # 🌟 上帝之手:强行兜底逻辑
372
+ # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁
373
+ if not rnd.any():
374
+ # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反,
375
+ # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。
376
+ # 这里统一下一个兜底,非常安全。
377
+ rand_pos = torch.randint(0, M, (1,), device=device)
378
+ rnd[rand_pos] = True
379
+
380
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
381
+ pmask[asst_idx] = rnd
382
+
383
+ noisy_embeds = clean_embeds.clone()
384
+ noisy_embeds[pmask] = mask_embed
385
+
386
+ # elif var_type == "random_masking" or var_type.startswith("coupled"):
387
+ # t = float(var_param)
388
+ # mask_seed = batch["mask_seed"][i]
389
+ # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
390
+ # rnd_cpu = torch.rand(M, generator=gen) < t
391
+
392
+ # if var_type == "coupled_comp":
393
+ # rnd_cpu = ~rnd_cpu
394
+
395
+ # pmask = torch.zeros(L, dtype=torch.bool, device=device)
396
+ # pmask[asst_idx] = rnd_cpu.to(device)
397
+
398
+ # noisy_embeds = clean_embeds.clone()
399
+ # noisy_embeds[pmask] = mask_embed
400
+
401
+ if not pmask.any():
402
+ print('************not pmask.any()**********')
403
+ continue
404
+
405
+ noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
406
+ labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
407
+ p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
408
+ attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
409
+ attn[-L:] = True
410
+ attn_list.append(attn)
411
+ reward_list.append(float(batch["reward"][i]))
412
+ valid_idxs.append(batch["idx"][i])
413
+
414
+ if len(noisy_list) == 0:
415
+ return None
416
+
417
+ noisy_batch = torch.stack(noisy_list, dim=0)
418
+ labels_batch = torch.stack(labels_list, dim=0)
419
+ p_mask_batch = torch.stack(p_mask_list, dim=0)
420
+ attn_batch = torch.stack(attn_list, dim=0)
421
+ reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32)
422
+
423
+ return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs
424
+
425
+
426
+ @torch.no_grad()
427
+ def compute_logp_old_tok_parallel(
428
+ accelerator,
429
+ model,
430
+ tokenizer,
431
+ image_processor,
432
+ dataset,
433
+ dl,
434
+ mask_id,
435
+ config):
436
+
437
+ model.eval()
438
+ logger.info("Computing reference log probabilities (old_lp) for all variants...")
439
+
440
+ for batch in dl:
441
+ out = prepare_batch_noisy_embeds_rl(
442
+ accelerator=accelerator,
443
+ model=model,
444
+ tokenizer=tokenizer,
445
+ image_processor=image_processor,
446
+ batch=batch,
447
+ mask_id=mask_id,
448
+ dataset_media_root=config.dataset.get("image_root", None),
449
+ )
450
+ if out is None:
451
+ continue
452
+
453
+ noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out
454
+
455
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True)
456
+ logits = model.lm_head(outputs.last_hidden_state).float()
457
+
458
+ log_probs = F.log_softmax(logits, dim=-1)
459
+ safe_labels = labels_batch.clone()
460
+ safe_labels[labels_batch == -100] = 0
461
+ tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
462
+
463
+ for i, idx in enumerate(valid_idxs):
464
+ dataset.logp_old_tok[idx] = tok_lp[i].float().cpu()
465
+
466
+ accelerator.wait_for_everyone()
467
+ model.train()
468
+
469
+
470
+ def save_checkpoint(model, tokenizer, config, accelerator, name):
471
+ output_dir = Path(config.experiment.project)
472
+ output_dir.mkdir(parents=True, exist_ok=True)
473
+
474
+ checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
475
+
476
+ if accelerator.is_main_process and checkpoints_total_limit is not None:
477
+ ckpts = sorted(
478
+ [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
479
+ key=lambda p: int(p.name.split("-")[1]),
480
+ )
481
+ if len(ckpts) >= checkpoints_total_limit:
482
+ to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
483
+ logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
484
+ for p in to_remove:
485
+ shutil.rmtree(p, ignore_errors=True)
486
+
487
+ save_base = output_dir / "ckpt"
488
+ save_base.mkdir(exist_ok=True)
489
+
490
+ model_to_save = accelerator.unwrap_model(model)
491
+ state_dict = accelerator.get_state_dict(model)
492
+
493
+ if accelerator.is_main_process:
494
+ model_to_save.save_pretrained(
495
+ save_base / name,
496
+ save_function=accelerator.save,
497
+ state_dict=state_dict,
498
+ safe_serialization=True,
499
+ )
500
+ tokenizer.save_pretrained(str(save_base / name))
501
+
502
+ metadata = {
503
+ "save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
504
+ }
505
+ with (save_base / "metadata.json").open("w") as f:
506
+ json.dump(metadata, f, indent=2)
507
+
508
+ logger.info(f"Saved model + tokenizer to {save_base / name}")
509
+
510
+
511
+ def main():
512
+ config = get_config()
513
+
514
+ project_name = config.experiment.project
515
+ if config.experiment.current_epoch == 1:
516
+ pretrained_model = config.model.pretrained_model
517
+ else:
518
+ pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name
519
+
520
+ if config.training.enable_tf32:
521
+ torch.backends.cuda.matmul.allow_tf32 = True
522
+ torch.backends.cudnn.benchmark = True
523
+ torch.backends.cudnn.deterministic = False
524
+
525
+ config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
526
+ accelerator = Accelerator(
527
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
528
+ mixed_precision=config.training.mixed_precision,
529
+ log_with='wandb',
530
+ project_dir=config.experiment.logging_dir,
531
+ split_batches=True,
532
+ )
533
+
534
+ logging.basicConfig(
535
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
536
+ datefmt="%m/%d/%Y %H:%M:%S",
537
+ level=logging.INFO,
538
+ )
539
+ logger.info(accelerator.state, main_process_only=False)
540
+ if accelerator.is_local_main_process:
541
+ set_verbosity_info()
542
+ else:
543
+ set_verbosity_error()
544
+
545
+ if accelerator.is_main_process:
546
+ resume_wandb_run = config.wandb.resume
547
+ run_id = config.wandb.get("run_id", None)
548
+ if run_id is None:
549
+ resume_wandb_run = False
550
+ run_id = wandb.util.generate_id()
551
+ config.wandb.run_id = run_id
552
+
553
+ wandb_init_kwargs = dict(
554
+ name=config.experiment.project,
555
+ id=run_id,
556
+ resume=resume_wandb_run,
557
+ entity=config.wandb.get("entity", None),
558
+ config_exclude_keys=[],
559
+ )
560
+ wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
561
+ wandb_config.pop("experiment.resume_from_checkpoint", None)
562
+
563
+ accelerator.init_trackers(
564
+ config.experiment.project,
565
+ config=wandb_config,
566
+ init_kwargs={"wandb": wandb_init_kwargs},
567
+ )
568
+
569
+ if accelerator.is_main_process:
570
+ os.makedirs(config.experiment.project, exist_ok=True)
571
+ config_path = Path(config.experiment.project) / "config.yaml"
572
+ logging.info(f"Saving config to {config_path}")
573
+ OmegaConf.save(config, config_path)
574
+
575
+ if config.training.seed is not None:
576
+ set_seed(config.training.seed)
577
+
578
+ logger.info("Loading LLaDA-V model and tokenizer")
579
+
580
+ tokenizer, model, image_processor, _ = load_pretrained_model(
581
+ pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
582
+ )
583
+ model = model.to(accelerator.device)
584
+
585
+ special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
586
+ num_new = tokenizer.add_special_tokens(special_tokens)
587
+ if num_new > 0:
588
+ model.resize_token_embeddings(len(tokenizer))
589
+ if tokenizer.pad_token is None:
590
+ tokenizer.pad_token = tokenizer.eos_token
591
+
592
+ if getattr(config.training, "gradient_checkpointing_enable", False):
593
+ try:
594
+ model.enable_input_require_grads()
595
+ except Exception:
596
+ pass
597
+ try:
598
+ model.gradient_checkpointing_enable()
599
+ except Exception:
600
+ pass
601
+
602
+ mask_id = 126336
603
+
604
+ optimizer_config = config.optimizer.params
605
+ no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
606
+ optimizer_grouped_parameters = [
607
+ {
608
+ "params": [p for n, p in model.named_parameters() if
609
+ p.requires_grad and not any(nd in n for nd in no_decay)],
610
+ "weight_decay": optimizer_config.weight_decay,
611
+ },
612
+ {
613
+ "params": [p for n, p in model.named_parameters() if
614
+ p.requires_grad and any(nd in n for nd in no_decay)],
615
+ "weight_decay": 0.0,
616
+ },
617
+ ]
618
+
619
+ optimizer_type = config.optimizer.name
620
+ if optimizer_type == "adamw":
621
+ optimizer = AdamW(
622
+ optimizer_grouped_parameters,
623
+ lr=optimizer_config.learning_rate,
624
+ betas=(optimizer_config.beta1, optimizer_config.beta2),
625
+ weight_decay=optimizer_config.weight_decay,
626
+ eps=optimizer_config.epsilon,
627
+ )
628
+ else:
629
+ raise ValueError(f"Optimizer {optimizer_type} not supported")
630
+
631
+
632
+ logger.info("Preprocessing Data")
633
+
634
+ with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f:
635
+ dataset_load = json.load(f)
636
+
637
+ dataset_lm = RLVariantDatasetLLADAV(dataset_load, config)
638
+
639
+ logger.info("Preparing model, optimizer and dataloaders")
640
+
641
+ total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
642
+ num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm)
643
+ num_train_epochs = config.training.num_train_epochs
644
+ max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
645
+
646
+ lr_scheduler = get_scheduler(
647
+ config.lr_scheduler.scheduler,
648
+ optimizer=optimizer,
649
+ num_training_steps=max_train_steps,
650
+ num_warmup_steps=config.lr_scheduler.params.warmup_steps,
651
+ min_lr_scale=config.lr_scheduler.params.min_lr_scale
652
+ )
653
+
654
+ train_dataloader_lm = DataLoader(
655
+ dataset_lm,
656
+ batch_size=config.training.batch_size_lm,
657
+ sampler=None,
658
+ collate_fn=collate_variants,
659
+ num_workers=0,
660
+ shuffle=False
661
+ )
662
+
663
+ model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare(
664
+ model, optimizer, lr_scheduler, train_dataloader_lm
665
+ )
666
+
667
+ logger.info("***** Running inference *****")
668
+ compute_logp_old_tok_parallel(
669
+ accelerator,
670
+ model,
671
+ tokenizer,
672
+ image_processor,
673
+ dataset_lm,
674
+ train_dataloader_lm,
675
+ mask_id=mask_id,
676
+ config=config
677
+ )
678
+
679
+ logger.info("***** Running training *****")
680
+ logger.info(f" Num variants = {len(dataset_lm)}")
681
+ logger.info(f" Num training steps = {max_train_steps}")
682
+ logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
683
+ logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
684
+ logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
685
+
686
+ first_epoch = 0
687
+ data_time_m = AverageMeter()
688
+ end = time.time()
689
+ grad_accum = config.training.gradient_accumulation_steps
690
+
691
+ def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok):
692
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
693
+ logits = model.lm_head(outputs.last_hidden_state).float()
694
+
695
+ B, T, V = logits.shape
696
+ adv = torch.as_tensor(adv, device=noisy_embeds.device).detach()
697
+
698
+ log_probs = F.log_softmax(logits, dim=-1)
699
+ safe_labels = labels.clone()
700
+ safe_labels[labels == -100] = 0
701
+ logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
702
+
703
+ ratio = torch.exp(logp_new_tok - logp_old_tok)
704
+ clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps)
705
+ adv_tok = adv.unsqueeze(1)
706
+
707
+ surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok)
708
+ surrogate_tok = surrogate_tok * p_mask_lm
709
+ num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1)
710
+ surrogate_tok = surrogate_tok.sum(dim=1) / num_mask
711
+ policy_loss = - (surrogate_tok.sum() / B)
712
+
713
+ # KL penalty
714
+ kl_loss = torch.tensor(0.0, device=policy_loss.device)
715
+ if config.training.beta > 0:
716
+ kl_seq = logp_new_tok - logp_old_tok
717
+ if config.training.use_kl_estimator_k3:
718
+ kl_seq = (-kl_seq).exp() - 1.0 + kl_seq
719
+ kl_seq = (kl_seq * p_mask_lm).sum(dim=1)
720
+ kl_loss = config.training.beta * kl_seq.sum() / B
721
+ total_loss = policy_loss + kl_loss
722
+ else:
723
+ total_loss = policy_loss
724
+
725
+ return total_loss
726
+
727
+ from tqdm.auto import tqdm
728
+
729
+ cprint("start rl training...", "red")
730
+ for epoch in range(first_epoch, num_train_epochs):
731
+ model.train()
732
+ progress_bar = tqdm(
733
+ train_dataloader_lm,
734
+ desc=f"Epoch {epoch+1}/{num_train_epochs}",
735
+ disable=not accelerator.is_local_main_process,
736
+ dynamic_ncols=True,
737
+ leave=True
738
+ )
739
+
740
+ step = 0
741
+ for step, batch in enumerate(progress_bar, start=1):
742
+ data_time_m.update(time.time() - end)
743
+
744
+ out = prepare_batch_noisy_embeds_rl(
745
+ accelerator=accelerator,
746
+ model=model,
747
+ tokenizer=tokenizer,
748
+ image_processor=image_processor,
749
+ batch=batch,
750
+ mask_id=mask_id,
751
+ dataset_media_root=config.dataset.get("image_root", None),
752
+ )
753
+
754
+ if out is None:
755
+ continue
756
+
757
+ noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out
758
+
759
+ # Align old log probabilities shape with current noisy_embeds
760
+ L_current = noisy_embeds.shape[1]
761
+ old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device)
762
+ for i, idx in enumerate(valid_idxs):
763
+ old_t = dataset_lm.logp_old_tok[idx]
764
+ l_min = min(L_current, old_t.shape[0])
765
+ old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device)
766
+
767
+ loss_lm = forward_process(
768
+ noisy_embeds=noisy_embeds,
769
+ labels=labels,
770
+ p_mask_lm=p_mask_lm,
771
+ attn_mask=attn_mask,
772
+ adv=rewards,
773
+ logp_old_tok=old_lp_batch
774
+ )
775
+
776
+ loss_lm = loss_lm / grad_accum
777
+ accelerator.backward(loss_lm)
778
+
779
+ if step % grad_accum == 0:
780
+ if config.training.max_grad_norm is not None:
781
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
782
+ optimizer.step()
783
+ lr_scheduler.step()
784
+ optimizer.zero_grad(set_to_none=True)
785
+ torch.cuda.empty_cache()
786
+
787
+ # Catch remaining gradients
788
+ if step % grad_accum != 0:
789
+ if config.training.max_grad_norm is not None:
790
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
791
+ optimizer.step()
792
+ lr_scheduler.step()
793
+ optimizer.zero_grad(set_to_none=True)
794
+ torch.cuda.empty_cache()
795
+
796
+ accelerator.wait_for_everyone()
797
+
798
+ save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
799
+ if config.experiment.current_epoch % config.experiment.save_every == 0:
800
+ save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}")
801
+
802
+ accelerator.end_training()
803
+
804
+
805
+ if __name__ == "__main__":
806
+ main()
wandb/run-20260323_031014-fmwqi7oc/files/config.yaml ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.25.0
4
+ code_path: code/train/rl_lladav.py
5
+ e:
6
+ 55pvqnbadfj4j7x2fc9exogdpb6eojxo:
7
+ args:
8
+ - config=configs/rl_lladav_video_reversal.yaml
9
+ - experiment.current_epoch=1
10
+ codePath: train/rl_lladav.py
11
+ codePathLocal: train/rl_lladav.py
12
+ cpu_count: 96
13
+ cpu_count_logical: 192
14
+ cudaVersion: "12.4"
15
+ disk:
16
+ /:
17
+ total: "806578724864"
18
+ used: "571723153408"
19
+ email: shixi_runqing@staff.weibo.com
20
+ executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10
21
+ git:
22
+ commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
23
+ remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
24
+ gpu: NVIDIA H800
25
+ gpu_count: 8
26
+ gpu_nvidia:
27
+ - architecture: Hopper
28
+ cudaCores: 16896
29
+ memoryTotal: "85520809984"
30
+ name: NVIDIA H800
31
+ uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
32
+ - architecture: Hopper
33
+ cudaCores: 16896
34
+ memoryTotal: "85520809984"
35
+ name: NVIDIA H800
36
+ uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
37
+ - architecture: Hopper
38
+ cudaCores: 16896
39
+ memoryTotal: "85520809984"
40
+ name: NVIDIA H800
41
+ uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
42
+ - architecture: Hopper
43
+ cudaCores: 16896
44
+ memoryTotal: "85520809984"
45
+ name: NVIDIA H800
46
+ uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
47
+ - architecture: Hopper
48
+ cudaCores: 16896
49
+ memoryTotal: "85520809984"
50
+ name: NVIDIA H800
51
+ uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
52
+ - architecture: Hopper
53
+ cudaCores: 16896
54
+ memoryTotal: "85520809984"
55
+ name: NVIDIA H800
56
+ uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
57
+ - architecture: Hopper
58
+ cudaCores: 16896
59
+ memoryTotal: "85520809984"
60
+ name: NVIDIA H800
61
+ uuid: GPU-2775489f-18c8-2328-8829-c46238575379
62
+ - architecture: Hopper
63
+ cudaCores: 16896
64
+ memoryTotal: "85520809984"
65
+ name: NVIDIA H800
66
+ uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
67
+ host: b1301b38c75f
68
+ memory:
69
+ total: "2164113813504"
70
+ os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
71
+ program: /data4/runqing/dLLM-RL/train/rl_lladav.py
72
+ python: CPython 3.10.19
73
+ root: /data4/runqing/dLLM-RL
74
+ startedAt: "2026-03-22T19:10:14.663079Z"
75
+ writerId: 55pvqnbadfj4j7x2fc9exogdpb6eojxo
76
+ m: []
77
+ python_version: 3.10.19
78
+ t:
79
+ "1":
80
+ - 1
81
+ - 5
82
+ - 11
83
+ - 41
84
+ - 49
85
+ - 53
86
+ - 71
87
+ - 98
88
+ "2":
89
+ - 1
90
+ - 5
91
+ - 11
92
+ - 41
93
+ - 49
94
+ - 51
95
+ - 53
96
+ - 71
97
+ - 98
98
+ "3":
99
+ - 13
100
+ - 14
101
+ "4": 3.10.19
102
+ "5": 0.25.0
103
+ "6": 4.52.4
104
+ "12": 0.25.0
105
+ "13": linux-x86_64
106
+ config:
107
+ value: configs/rl_lladav_video_reversal.yaml
108
+ dataset.data_type:
109
+ value: video_reverse
110
+ dataset.optimization_data:
111
+ value: rl_data
112
+ dataset.train_dataset:
113
+ value: video_reversal_rl_train
114
+ evaluation.batch_size:
115
+ value: 1
116
+ evaluation.block_size:
117
+ value: 32
118
+ evaluation.data_type:
119
+ value: video_reverse
120
+ evaluation.dynamic_threshold:
121
+ value: 0.9
122
+ evaluation.eval_dataset:
123
+ value: video_reversal_rl_train
124
+ evaluation.further_horizon:
125
+ value: 128
126
+ evaluation.if_eval:
127
+ value: true
128
+ evaluation.max_gen_length:
129
+ value: 256
130
+ evaluation.num_response_per_task:
131
+ value: 3
132
+ evaluation.remasking_strategy.0:
133
+ value: low_confidence_static
134
+ evaluation.remasking_strategy.1:
135
+ value: low_confidence_dynamic
136
+ evaluation.steps:
137
+ value: 256
138
+ evaluation.target:
139
+ value: confidence
140
+ evaluation.temperature:
141
+ value: 0.1
142
+ evaluation.use_cache:
143
+ value: true
144
+ execute.num_chunk:
145
+ value: 128
146
+ experiment.current_epoch:
147
+ value: 1
148
+ experiment.deepspeed_file:
149
+ value: 1_node_8_gpus_deepspeed_zero2_rl
150
+ experiment.eval_every:
151
+ value: 5
152
+ experiment.function:
153
+ value: train
154
+ experiment.logging_dir:
155
+ value: rl_lladav_video_reversal/logs
156
+ experiment.node_index:
157
+ value: 0
158
+ experiment.num_node:
159
+ value: 1
160
+ experiment.project:
161
+ value: rl_lladav_video_reversal
162
+ experiment.save_every:
163
+ value: 10
164
+ experiment.start_from_scratch:
165
+ value: true
166
+ experiment.total_step:
167
+ value: 160
168
+ lr_scheduler.params.learning_rate:
169
+ value: 1e-06
170
+ lr_scheduler.params.min_lr_scale:
171
+ value: 1
172
+ lr_scheduler.params.warmup_steps:
173
+ value: 0
174
+ lr_scheduler.scheduler:
175
+ value: cosine
176
+ model.image_resolution:
177
+ value: 512
178
+ model.model_base:
179
+ value: lladav
180
+ model.optimized_name:
181
+ value: optimized
182
+ model.pretrained_model:
183
+ value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
184
+ model.vq_model_path:
185
+ value: /data4/runqing/hf/magvitv2
186
+ optimizer.name:
187
+ value: adamw
188
+ optimizer.params.beta1:
189
+ value: 0.9
190
+ optimizer.params.beta2:
191
+ value: 0.999
192
+ optimizer.params.epsilon:
193
+ value: 1e-08
194
+ optimizer.params.learning_rate:
195
+ value: 1e-06
196
+ optimizer.params.scale_lr:
197
+ value: false
198
+ optimizer.params.weight_decay:
199
+ value: 0
200
+ reward.answer_must_in_box:
201
+ value: true
202
+ reward.strict_len_check:
203
+ value: false
204
+ rollout.batch_size:
205
+ value: 1
206
+ rollout.block_size:
207
+ value: 32
208
+ rollout.dynamic_threshold:
209
+ value: 0.9
210
+ rollout.further_horizon:
211
+ value: 128
212
+ rollout.max_gen_length:
213
+ value: 128
214
+ rollout.num_response_per_task:
215
+ value: 8
216
+ rollout.num_task_per_step:
217
+ value: 128
218
+ rollout.remasking_strategy:
219
+ value: low_confidence_dynamic
220
+ rollout.steps:
221
+ value: 128
222
+ rollout.target:
223
+ value: confidence
224
+ rollout.temperature:
225
+ value: 0.8
226
+ rollout.use_cache:
227
+ value: true
228
+ system.HF_HOME:
229
+ value: null
230
+ system.HTTP_PROXY:
231
+ value: null
232
+ system.base_dir:
233
+ value: /data4/runqing/dLLM-RL
234
+ system.env_name:
235
+ value: /data8/runqing/miniconda/envs/dllm-rl
236
+ system.envs_dir:
237
+ value: /data8/runqing/miniconda/envs/dllm-rl
238
+ training.batch_size_lm:
239
+ value: 2
240
+ training.beta:
241
+ value: 0.01
242
+ training.enable_tf32:
243
+ value: true
244
+ training.eps:
245
+ value: 0.2
246
+ training.gradient_accumulation_steps:
247
+ value: 64
248
+ training.lower_p:
249
+ value: 0.35
250
+ training.mask_times_per_sample:
251
+ value: 2
252
+ training.max_gen_length:
253
+ value: 128
254
+ training.max_grad_norm:
255
+ value: 1
256
+ training.max_prompt_len:
257
+ value: 512
258
+ training.method:
259
+ value: random_masking
260
+ training.mixed_precision:
261
+ value: bf16
262
+ training.num_train_epochs:
263
+ value: 1
264
+ training.post_num:
265
+ value: 1
266
+ training.seed:
267
+ value: 10086
268
+ training.shrink:
269
+ value: 8
270
+ training.upper_p:
271
+ value: 0.9
272
+ training.use_kl_estimator_k3:
273
+ value: true
274
+ wandb.entity:
275
+ value: null
276
+ wandb.resume:
277
+ value: auto
278
+ wandb.run_id:
279
+ value: fmwqi7oc
wandb/run-20260323_031014-fmwqi7oc/files/diff.patch ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..5701c74 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,62 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [],
158
+ + "source": [
159
+ + "import json\n",
160
+ + "\n",
161
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
162
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
163
+ + "save_data_list = []\n",
164
+ + "save_num = 0\n",
165
+ + "with open(data_path, 'r') as fin, \\\n",
166
+ + " open(save_path, 'w') as fout:\n",
167
+ + " raw_data = json.load(fin)\n",
168
+ + " for data in raw_data:\n",
169
+ + " if save_num < 100:\n",
170
+ + " save_data_list.append(data)\n",
171
+ + " save_num += 1\n",
172
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
173
+ + " print(len(save_data_list))\n"
174
+ + ]
175
+ }
176
+ ],
177
+ "metadata": {
178
+ @@ -85,7 +141,7 @@
179
+ "name": "python",
180
+ "nbconvert_exporter": "python",
181
+ "pygments_lexer": "ipython3",
182
+ - "version": "3.10.18"
183
+ + "version": "3.10.19"
184
+ }
185
+ },
186
+ "nbformat": 4,
187
+ diff --git a/rl_v.py b/rl_v.py
188
+ index 8f2c0f1..79d2e5b 100644
189
+ --- a/rl_v.py
190
+ +++ b/rl_v.py
191
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
192
+ def env_prefix(env) -> str:
193
+ return (
194
+ "source ~/.bashrc && "
195
+ - f"source activate {env} && "
196
+ + "source /data8/runqing/miniconda/bin/activate && "
197
+ + "conda init && "
198
+ + f"conda activate {env} && "
199
+ )
200
+
201
+ config = get_config()
202
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
203
+ index 4068fc8..d284a23 100644
204
+ --- a/sample/llava/conversation.py
205
+ +++ b/sample/llava/conversation.py
206
+ @@ -99,7 +99,7 @@ class Conversation:
207
+ if self.version == "llama_v3":
208
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
209
+ elif self.version == "llava_llada":
210
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
211
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
212
+ else:
213
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
214
+ chat_template_messages = [{"role": "system", "content": self.system}]
215
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
216
+ index 3db5302..96e33b5 100644
217
+ --- a/sample/llava/model/builder.py
218
+ +++ b/sample/llava/model/builder.py
219
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
220
+ elif "llada" in model_name.lower():
221
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
222
+
223
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
224
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
225
+ +
226
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
227
+ if customized_config is None:
228
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
229
+ else:
230
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
231
+ index 4068fc8..d284a23 100644
232
+ --- a/train/llava/conversation.py
233
+ +++ b/train/llava/conversation.py
234
+ @@ -99,7 +99,7 @@ class Conversation:
235
+ if self.version == "llama_v3":
236
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
237
+ elif self.version == "llava_llada":
238
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
239
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
240
+ else:
241
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
242
+ chat_template_messages = [{"role": "system", "content": self.system}]
243
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
244
+ index 3db5302..aafab95 100644
245
+ --- a/train/llava/model/builder.py
246
+ +++ b/train/llava/model/builder.py
247
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
248
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
249
+ elif "llada" in model_name.lower():
250
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
251
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
252
+
253
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
254
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
255
+ if customized_config is None:
256
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
257
+ else:
258
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
259
+ index 3da28dc..8b17da3 100644
260
+ --- a/train/llava/model/language_model/llava_llada.py
261
+ +++ b/train/llava/model/language_model/llava_llada.py
262
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
263
+ dpo_forward: Optional[bool] = None,
264
+ cache_position=None,
265
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
266
+ -
267
+ + conversation_ids = None
268
+ if inputs_embeds is None and attention_mask is not None:
269
+ # donate multi-dialogue
270
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..5701c74 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,62 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [],
158
+ + "source": [
159
+ + "import json\n",
160
+ + "\n",
161
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n",
162
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
163
+ + "save_data_list = []\n",
164
+ + "save_num = 0\n",
165
+ + "with open(data_path, 'r') as fin, \\\n",
166
+ + " open(save_path, 'w') as fout:\n",
167
+ + " raw_data = json.load(fin)\n",
168
+ + " for data in raw_data:\n",
169
+ + " if save_num < 100:\n",
170
+ + " save_data_list.append(data)\n",
171
+ + " save_num += 1\n",
172
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
173
+ + " print(len(save_data_list))\n"
174
+ + ]
175
+ }
176
+ ],
177
+ "metadata": {
178
+ @@ -85,7 +141,7 @@
179
+ "name": "python",
180
+ "nbconvert_exporter": "python",
181
+ "pygments_lexer": "ipython3",
182
+ - "version": "3.10.18"
183
+ + "version": "3.10.19"
184
+ }
185
+ },
186
+ "nbformat": 4,
187
+ diff --git a/rl_v.py b/rl_v.py
188
+ index 8f2c0f1..79d2e5b 100644
189
+ --- a/rl_v.py
190
+ +++ b/rl_v.py
191
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
192
+ def env_prefix(env) -> str:
193
+ return (
194
+ "source ~/.bashrc && "
195
+ - f"source activate {env} && "
196
+ + "source /data8/runqing/miniconda/bin/activate && "
197
+ + "conda init && "
198
+ + f"conda activate {env} && "
199
+ )
200
+
201
+ config = get_config()
202
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
203
+ index 4068fc8..d284a23 100644
204
+ --- a/sample/llava/conversation.py
205
+ +++ b/sample/llava/conversation.py
206
+ @@ -99,7 +99,7 @@ class Conversation:
207
+ if self.version == "llama_v3":
208
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
209
+ elif self.version == "llava_llada":
210
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
211
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
212
+ else:
213
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
214
+ chat_template_messages = [{"role": "system", "content": self.system}]
215
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
216
+ index 3db5302..96e33b5 100644
217
+ --- a/sample/llava/model/builder.py
218
+ +++ b/sample/llava/model/builder.py
219
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
220
+ elif "llada" in model_name.lower():
221
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
222
+
223
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
224
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
225
+ +
226
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
227
+ if customized_config is None:
228
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
229
+ else:
230
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
231
+ index 4068fc8..d284a23 100644
232
+ --- a/train/llava/conversation.py
233
+ +++ b/train/llava/conversation.py
234
+ @@ -99,7 +99,7 @@ class Conversation:
235
+ if self.version == "llama_v3":
236
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
237
+ elif self.version == "llava_llada":
238
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
239
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
240
+ else:
241
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
242
+ chat_template_messages = [{"role": "system", "content": self.system}]
243
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
244
+ index 3db5302..aafab95 100644
245
+ --- a/train/llava/model/builder.py
246
+ +++ b/train/llava/model/builder.py
247
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
248
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
249
+ elif "llada" in model_name.lower():
250
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
251
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
252
+
253
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
254
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
255
+ if customized_config is None:
256
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
257
+ else:
258
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
259
+ index 3da28dc..8b17da3 100644
260
+ --- a/train/llava/model/language_model/llava_llada.py
261
+ +++ b/train/llava/model/language_model/llava_llada.py
262
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
263
+ dpo_forward: Optional[bool] = None,
264
+ cache_position=None,
265
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
266
+ -
267
+ + conversation_ids = None
268
+ if inputs_embeds is None and attention_mask is not None:
269
+ # donate multi-dialogue
270
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
wandb/run-20260323_031014-fmwqi7oc/files/output.log ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 03/23/2026 03:10:19 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml
2
+ 03/23/2026 03:10:19 - INFO - __main__ - Loading LLaDA-V model and tokenizer
3
+ Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
4
+ Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
5
+ Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 5.68s/it]
6
+ Rank 0: Model Class: LlavaLLaDAModelLM
7
+ 03/23/2026 03:10:45 - INFO - __main__ - Preprocessing Data
8
+ 03/23/2026 03:10:45 - INFO - __main__ - Preparing model, optimizer and dataloaders
9
+ /data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
10
+ If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
11
+ warnings.warn(
12
+ Before initializing optimizer states
13
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
14
+ CPU Virtual Memory: used = 160.24 GB, percent = 8.0%
15
+ After initializing optimizer states
16
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
17
+ CPU Virtual Memory: used = 186.43 GB, percent = 9.2%
18
+ After initializing ZeRO optimizer
19
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
20
+ CPU Virtual Memory: used = 189.08 GB, percent = 9.4%
21
+ 03/23/2026 03:12:00 - INFO - __main__ - ***** Running inference *****
22
+ 03/23/2026 03:12:00 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants...
23
+ 03/23/2026 03:14:03 - INFO - __main__ - ***** Running training *****
24
+ 03/23/2026 03:14:03 - INFO - __main__ - Num variants = 384
25
+ 03/23/2026 03:14:03 - INFO - __main__ - Num training steps = 2
26
+ 03/23/2026 03:14:03 - INFO - __main__ - Instantaneous batch size per device = 2
27
+ 03/23/2026 03:14:03 - INFO - __main__ - Total train batch size (parallel * accum) = 1024
28
+ 03/23/2026 03:14:03 - INFO - __main__ - Gradient Accumulation steps = 64
29
+ start rl training...
30
+ Epoch 1/1: 8%|▊ | 2/24 [00:32<06:02, 16.47s/it]
31
+ Traceback (most recent call last):
32
+ File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 806, in <module>
33
+ main()
34
+ File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 767, in main
35
+ loss_lm = forward_process(
36
+ File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 692, in forward_process
37
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
38
+ File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
39
+ return self._call_impl(*args, **kwargs)
40
+ File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
41
+ return forward_call(*args, **kwargs)
42
+ File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 1027, in forward
43
+ layer_outputs = decoder_layer(
44
+ File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
45
+ return self._call_impl(*args, **kwargs)
46
+ File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
47
+ return forward_call(*args, **kwargs)
48
+ File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 759, in forward
49
+ hidden_states = self.post_attention_layernorm(hidden_states)
50
+ File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
51
+ return self._call_impl(*args, **kwargs)
52
+ File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
53
+ return forward_call(*args, **kwargs)
54
+ File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 89, in forward
55
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
56
+ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 214.00 MiB. GPU 0 has a total capacity of 79.11 GiB of which 208.81 MiB is free. Process 2750357 has 78.90 GiB memory in use. Of the allocated memory 73.41 GiB is allocated by PyTorch, and 1.34 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
57
+ [rank0]: Traceback (most recent call last):
58
+ [rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 806, in <module>
59
+ [rank0]: main()
60
+ [rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 767, in main
61
+ [rank0]: loss_lm = forward_process(
62
+ [rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 692, in forward_process
63
+ [rank0]: outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
64
+ [rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
65
+ [rank0]: return self._call_impl(*args, **kwargs)
66
+ [rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
67
+ [rank0]: return forward_call(*args, **kwargs)
68
+ [rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 1027, in forward
69
+ [rank0]: layer_outputs = decoder_layer(
70
+ [rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
71
+ [rank0]: return self._call_impl(*args, **kwargs)
72
+ [rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
73
+ [rank0]: return forward_call(*args, **kwargs)
74
+ [rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 759, in forward
75
+ [rank0]: hidden_states = self.post_attention_layernorm(hidden_states)
76
+ [rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
77
+ [rank0]: return self._call_impl(*args, **kwargs)
78
+ [rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
79
+ [rank0]: return forward_call(*args, **kwargs)
80
+ [rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 89, in forward
81
+ [rank0]: variance = hidden_states.pow(2).mean(-1, keepdim=True)
82
+ [rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 214.00 MiB. GPU 0 has a total capacity of 79.11 GiB of which 208.81 MiB is free. Process 2750357 has 78.90 GiB memory in use. Of the allocated memory 73.41 GiB is allocated by PyTorch, and 1.34 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ transformers==4.52.4
2
+ nvidia-nvtx-cu12==12.4.127
3
+ deepspeed==0.18.6
4
+ typing-inspection==0.4.2
5
+ scipy==1.15.3
6
+ MarkupSafe==3.0.3
7
+ nvidia-cuda-runtime-cu12==12.4.127
8
+ yarl==1.22.0
9
+ tornado==6.5.4
10
+ nvidia-curand-cu12==10.3.5.147
11
+ nest-asyncio==1.6.0
12
+ safetensors==0.7.0
13
+ async-timeout==5.0.1
14
+ typing_extensions==4.15.0
15
+ asttokens==3.0.1
16
+ cuda-python==13.1.1
17
+ executing==2.2.1
18
+ nvidia-cudnn-frontend==1.18.0
19
+ comm==0.2.3
20
+ tokenizers==0.21.4
21
+ six==1.17.0
22
+ Pygments==2.19.2
23
+ nvidia-cusparselt-cu12==0.6.2
24
+ apache-tvm-ffi==0.1.8.post2
25
+ flash_attn==2.7.4.post1
26
+ tqdm==4.67.3
27
+ tensorboard-data-server==0.7.2
28
+ annotated-types==0.7.0
29
+ math-verify==0.9.0
30
+ idna==3.11
31
+ regex==2026.1.15
32
+ nvidia-cutlass-dsl-libs-base==4.4.0
33
+ flashinfer-python==0.6.3
34
+ click==8.3.1
35
+ pytz==2025.2
36
+ wheel==0.46.3
37
+ ipykernel==7.2.0
38
+ stack-data==0.6.3
39
+ absl-py==2.4.0
40
+ wandb==0.25.0
41
+ pandas==2.3.3
42
+ termcolor==3.3.0
43
+ sympy==1.13.1
44
+ joblib==1.5.3
45
+ prompt_toolkit==3.0.52
46
+ attrs==25.4.0
47
+ nvidia-cusolver-cu12==11.6.1.9
48
+ xxhash==3.6.0
49
+ aiohappyeyeballs==2.6.1
50
+ python-dateutil==2.9.0.post0
51
+ frozenlist==1.8.0
52
+ nvidia-ml-py==13.590.48
53
+ multiprocess==0.70.18
54
+ decorator==5.2.1
55
+ cuda-pathfinder==1.3.4
56
+ multidict==6.7.1
57
+ pure_eval==0.2.3
58
+ nvidia-cufft-cu12==11.2.1.3
59
+ omegaconf==2.3.0
60
+ PyYAML==6.0.3
61
+ debugpy==1.8.20
62
+ psutil==7.2.2
63
+ ninja==1.13.0
64
+ nvidia-cublas-cu12==12.4.5.8
65
+ triton==3.2.0
66
+ antlr4-python3-runtime==4.9.3
67
+ peft==0.18.1
68
+ aiosignal==1.4.0
69
+ accelerate==1.12.0
70
+ liger_kernel==0.7.0
71
+ anyio==4.12.1
72
+ decord==0.6.0
73
+ h11==0.16.0
74
+ urllib3==2.6.3
75
+ wcwidth==0.6.0
76
+ networkx==3.4.2
77
+ huggingface_hub==0.36.2
78
+ tzdata==2025.3
79
+ xformers==0.0.29.post3
80
+ tabulate==0.9.0
81
+ certifi==2026.1.4
82
+ tensorboard==2.20.0
83
+ matplotlib-inline==0.2.1
84
+ pip==26.0.1
85
+ smmap==5.0.2
86
+ nvidia-cuda-cupti-cu12==12.4.127
87
+ Markdown==3.10.2
88
+ nvidia-nccl-cu12==2.21.5
89
+ aiohttp==3.13.3
90
+ scikit-learn==1.7.2
91
+ nvidia-cutlass-dsl==4.4.0
92
+ setuptools==82.0.0
93
+ jupyter_core==5.9.1
94
+ asyncio==4.0.0
95
+ einops==0.8.2
96
+ pydantic==2.12.5
97
+ av==16.1.0
98
+ protobuf==6.33.5
99
+ evaluate==0.4.6
100
+ Jinja2==3.1.6
101
+ exceptiongroup==1.3.1
102
+ ipython==8.38.0
103
+ sentry-sdk==2.53.0
104
+ httpx==0.28.1
105
+ msgpack==1.1.2
106
+ platformdirs==4.9.2
107
+ hjson==3.1.0
108
+ torchvision==0.21.0
109
+ parso==0.8.6
110
+ cuda-bindings==13.1.1
111
+ pexpect==4.9.0
112
+ threadpoolctl==3.6.0
113
+ pyarrow==23.0.1
114
+ jedi==0.19.2
115
+ nvidia-cuda-nvrtc-cu12==12.4.127
116
+ Werkzeug==3.1.5
117
+ fsspec==2025.10.0
118
+ pydantic_core==2.41.5
119
+ GitPython==3.1.46
120
+ propcache==0.4.1
121
+ mpmath==1.3.0
122
+ nvidia-nvjitlink-cu12==12.4.127
123
+ nvidia-cusparse-cu12==12.3.1.170
124
+ hf-xet==1.2.0
125
+ charset-normalizer==3.4.4
126
+ dill==0.4.0
127
+ datasets==4.5.0
128
+ py-cpuinfo==9.0.0
129
+ sentencepiece==0.2.1
130
+ latex2sympy2_extended==1.11.0
131
+ jupyter_client==8.8.0
132
+ pyzmq==27.1.0
133
+ numpy==2.2.6
134
+ gitdb==4.0.12
135
+ grpcio==1.78.0
136
+ packaging==26.0
137
+ requests==2.32.5
138
+ nvidia-cudnn-cu12==9.1.0.70
139
+ httpcore==1.0.9
140
+ traitlets==5.14.3
141
+ filelock==3.24.2
142
+ ptyprocess==0.7.0
143
+ pillow==12.1.1
144
+ torch==2.6.0
145
+ more-itertools==10.8.0
146
+ wheel==0.46.3
147
+ jaraco.text==4.0.0
148
+ zipp==3.23.0
149
+ autocommand==2.2.2
150
+ jaraco.context==6.1.0
151
+ jaraco.functools==4.4.0
152
+ tomli==2.4.0
153
+ backports.tarfile==1.2.0
154
+ importlib_metadata==8.7.1
155
+ platformdirs==4.4.0
156
+ packaging==26.0
wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.19",
4
+ "startedAt": "2026-03-22T19:10:14.663079Z",
5
+ "args": [
6
+ "config=configs/rl_lladav_video_reversal.yaml",
7
+ "experiment.current_epoch=1"
8
+ ],
9
+ "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py",
10
+ "codePath": "train/rl_lladav.py",
11
+ "codePathLocal": "train/rl_lladav.py",
12
+ "git": {
13
+ "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
14
+ "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
15
+ },
16
+ "email": "shixi_runqing@staff.weibo.com",
17
+ "root": "/data4/runqing/dLLM-RL",
18
+ "host": "b1301b38c75f",
19
+ "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H800",
23
+ "gpu_count": 8,
24
+ "disk": {
25
+ "/": {
26
+ "total": "806578724864",
27
+ "used": "571723153408"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "2164113813504"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H800",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
40
+ },
41
+ {
42
+ "name": "NVIDIA H800",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
47
+ },
48
+ {
49
+ "name": "NVIDIA H800",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
54
+ },
55
+ {
56
+ "name": "NVIDIA H800",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
61
+ },
62
+ {
63
+ "name": "NVIDIA H800",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
68
+ },
69
+ {
70
+ "name": "NVIDIA H800",
71
+ "memoryTotal": "85520809984",
72
+ "cudaCores": 16896,
73
+ "architecture": "Hopper",
74
+ "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
75
+ },
76
+ {
77
+ "name": "NVIDIA H800",
78
+ "memoryTotal": "85520809984",
79
+ "cudaCores": 16896,
80
+ "architecture": "Hopper",
81
+ "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
82
+ },
83
+ {
84
+ "name": "NVIDIA H800",
85
+ "memoryTotal": "85520809984",
86
+ "cudaCores": 16896,
87
+ "architecture": "Hopper",
88
+ "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
89
+ }
90
+ ],
91
+ "cudaVersion": "12.4",
92
+ "writerId": "55pvqnbadfj4j7x2fc9exogdpb6eojxo"
93
+ }
wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb":{"runtime":256},"_runtime":256}
wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-23T03:10:14.734689708+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpxva6clnc/port-3523574.txt","pid":3523574,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-03-23T03:10:14.735635443+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3523574}
3
+ {"time":"2026-03-23T03:10:14.735628068+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3523574-3524049-679808348/socket","Net":"unix"}}
4
+ {"time":"2026-03-23T03:10:14.915970148+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-03-23T03:10:14.923903515+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmwqi7oc","id":"1(@)"}
6
+ {"time":"2026-03-23T03:10:19.250521705+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmwqi7oc","id":"1(@)"}
7
+ {"time":"2026-03-23T03:10:24.490584047+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"srafqf41avyr"}
8
+ {"time":"2026-03-23T03:14:36.354854515+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
9
+ {"time":"2026-03-23T03:14:36.354904386+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
10
+ {"time":"2026-03-23T03:14:36.354956667+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
11
+ {"time":"2026-03-23T03:14:36.354918741+08:00","level":"INFO","msg":"server is shutting down"}
12
+ {"time":"2026-03-23T03:14:36.355031381+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3523574-3524049-679808348/socket","Net":"unix"}}
13
+ {"time":"2026-03-23T03:14:36.574170314+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
14
+ {"time":"2026-03-23T03:14:36.574226901+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
15
+ {"time":"2026-03-23T03:14:36.574272235+08:00","level":"INFO","msg":"server is closed"}
wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-23T03:10:14.923991754+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
2
+ {"time":"2026-03-23T03:10:19.249985082+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
3
+ {"time":"2026-03-23T03:10:19.250206083+08:00","level":"INFO","msg":"stream: created new stream","id":"fmwqi7oc"}
4
+ {"time":"2026-03-23T03:10:19.250314544+08:00","level":"INFO","msg":"handler: started","stream_id":"fmwqi7oc"}
5
+ {"time":"2026-03-23T03:10:19.250506873+08:00","level":"INFO","msg":"stream: started","id":"fmwqi7oc"}
6
+ {"time":"2026-03-23T03:10:19.250555978+08:00","level":"INFO","msg":"writer: started","stream_id":"fmwqi7oc"}
7
+ {"time":"2026-03-23T03:10:19.250551745+08:00","level":"INFO","msg":"sender: started","stream_id":"fmwqi7oc"}
8
+ {"time":"2026-03-23T03:10:19.253219433+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
9
+ {"time":"2026-03-23T03:10:24.455218043+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=424ac51772e4e6aeadd1916a4c5be0ccd13e3e923cbb73fc604d2577f3cda919"}
10
+ {"time":"2026-03-23T03:10:24.455395899+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=810d6278f7888329964e292ae4b8d352346389dba5b1e0d16660a92138f8fa0b"}
11
+ {"time":"2026-03-23T03:10:24.455410759+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json</Resource><RequestId>189F3F86588581D5</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=424ac51772e4e6aeadd1916a4c5be0ccd13e3e923cbb73fc604d2577f3cda919, Size: 2635}"}
12
+ {"time":"2026-03-23T03:10:24.455507161+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt</Resource><RequestId>189F3F8658979FBE</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=810d6278f7888329964e292ae4b8d352346389dba5b1e0d16660a92138f8fa0b, Size: 2966}"}
13
+ {"time":"2026-03-23T03:10:24.455771816+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8742bb45572bf89449139797076cdb743a0f0e84b1b2d691043a6f8cab81636"}
14
+ {"time":"2026-03-23T03:10:24.455883557+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189F3F8658A48B88</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8742bb45572bf89449139797076cdb743a0f0e84b1b2d691043a6f8cab81636, Size: 11427}"}
15
+ {"time":"2026-03-23T03:10:24.458949814+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d60641ef66f42e5ff9c0558786e2031e714aa06ed1c87f70bfd6432a5e2bce"}
16
+ {"time":"2026-03-23T03:10:24.459156615+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch</Resource><RequestId>189F3F8658B03704</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d60641ef66f42e5ff9c0558786e2031e714aa06ed1c87f70bfd6432a5e2bce, Size: 11427}"}
17
+ {"time":"2026-03-23T03:10:24.460144111+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b854582f73a02708bfcd8010e9db9ec37b13d12c64988b1b7d8d7a37414bc0e1"}
18
+ {"time":"2026-03-23T03:10:24.460213159+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py</Resource><RequestId>189F3F865877B3DB</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b854582f73a02708bfcd8010e9db9ec37b13d12c64988b1b7d8d7a37414bc0e1, Size: 31804}"}
19
+ {"time":"2026-03-23T03:14:36.354902346+08:00","level":"INFO","msg":"stream: closing","id":"fmwqi7oc"}
20
+ {"time":"2026-03-23T03:14:36.433198106+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=884fe6b1bfe554b7c975118d664e5b487c0bf178608ab506bd9a0e45c348c413"}
21
+ {"time":"2026-03-23T03:14:36.433310818+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log</Resource><RequestId>189F3FC103A8B9D5</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=884fe6b1bfe554b7c975118d664e5b487c0bf178608ab506bd9a0e45c348c413, Size: 7319}"}
22
+ {"time":"2026-03-23T03:14:36.437455015+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba2517cac97729d5e42d029ad94e53f0fdce01e5c8382b91f9f83fad1535086d"}
23
+ {"time":"2026-03-23T03:14:36.437526478+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json</Resource><RequestId>189F3FC103D3486C</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba2517cac97729d5e42d029ad94e53f0fdce01e5c8382b91f9f83fad1535086d, Size: 41}"}
24
+ {"time":"2026-03-23T03:14:36.437757914+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a8c9b732a52ea76e7908bfd3e2f5b5536c95c03abe7dc4d1913e42c397ea578d"}
25
+ {"time":"2026-03-23T03:14:36.437912275+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml</Resource><RequestId>189F3FC103DD3840</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a8c9b732a52ea76e7908bfd3e2f5b5536c95c03abe7dc4d1913e42c397ea578d, Size: 7766}"}
26
+ {"time":"2026-03-23T03:14:36.438391389+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
27
+ {"time":"2026-03-23T03:14:36.573284944+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmwqi7oc"}
28
+ {"time":"2026-03-23T03:14:36.573493717+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmwqi7oc"}
29
+ {"time":"2026-03-23T03:14:36.573524005+08:00","level":"INFO","msg":"stream: closed","id":"fmwqi7oc"}
wandb/run-20260323_031014-fmwqi7oc/logs/debug.log ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
2
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Configure stats pid to 3523574
3
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/logs/debug.log
5
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log
6
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():844] calling init triggers
7
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
+ config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}}
9
+ 2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():892] starting backend
10
+ 2026-03-23 03:10:14,916 INFO MainThread:3523574 [wandb_init.py:init():895] sending inform_init request
11
+ 2026-03-23 03:10:14,920 INFO MainThread:3523574 [wandb_init.py:init():903] backend started and connected
12
+ 2026-03-23 03:10:14,923 INFO MainThread:3523574 [wandb_init.py:init():973] updated telemetry
13
+ 2026-03-23 03:10:14,928 INFO MainThread:3523574 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
+ 2026-03-23 03:10:19,377 INFO MainThread:3523574 [wandb_init.py:init():1042] starting run threads in backend
15
+ 2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_console_start():2524] atexit reg
16
+ 2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-03-23 03:10:19,489 INFO MainThread:3523574 [wandb_init.py:init():1082] run started, returning control to user process
20
+ 2026-03-23 03:10:19,490 INFO MainThread:3523574 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmwqi7oc', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 160, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 64, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_train', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 256, 'evaluation.max_gen_length': 256, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'}
21
+ 2026-03-23 03:14:36,355 INFO wandb-AsyncioManager-main:3523574 [service_client.py:_forward_responses():134] Reached EOF.
22
+ 2026-03-23 03:14:36,355 INFO wandb-AsyncioManager-main:3523574 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb ADDED
Binary file (86.3 kB). View file
 
wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py ADDED
@@ -0,0 +1,815 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
4
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
5
+ from datetime import timedelta
6
+ import json
7
+ import logging
8
+ import math
9
+ import shutil
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Dict, List, Any
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from torch.optim import AdamW
18
+ from torch.utils.data import Dataset, DataLoader
19
+
20
+ from omegaconf import OmegaConf
21
+ import wandb
22
+
23
+ from accelerate import Accelerator
24
+ from accelerate.logging import get_logger
25
+ from accelerate.utils import set_seed
26
+
27
+ from train.utils import get_config, flatten_omega_conf, AverageMeter
28
+ from models.lr_schedulers import get_scheduler
29
+ from models.logging import set_verbosity_info, set_verbosity_error
30
+
31
+ from transformers.utils import logging as hf_logging
32
+ hf_logging.set_verbosity_error()
33
+
34
+ from llava.model.builder import load_pretrained_model
35
+ from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord
36
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
37
+ from llava.conversation import conv_templates
38
+
39
+ from termcolor import cprint
40
+
41
+ logger = get_logger(__name__, log_level="INFO")
42
+
43
+
44
+ def collapse_k_unique(lst, k: int):
45
+ if k <= 0:
46
+ raise ValueError("k must be > 0")
47
+ uniq = sorted(set(lst))
48
+ mapping = {}
49
+ n = len(uniq)
50
+ for idx, val in enumerate(uniq):
51
+ group = idx // k
52
+ end_idx = min((group + 1) * k - 1, n - 1)
53
+ rep = uniq[end_idx]
54
+ mapping[val] = rep
55
+ return [mapping[x] for x in lst]
56
+
57
+
58
+ def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
59
+ import copy
60
+ conv = copy.deepcopy(conv_templates["llava_llada"])
61
+ conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
62
+ conv.append_message(conv.roles[1], response_text)
63
+ return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
64
+
65
+
66
+ def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
67
+ conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
68
+ ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
69
+ return ids.squeeze(0)
70
+
71
+
72
+ def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
73
+ from PIL import Image
74
+ from types import SimpleNamespace
75
+
76
+ if isinstance(model_cfg, dict):
77
+ model_cfg = SimpleNamespace(**model_cfg)
78
+
79
+ if media_type == "image":
80
+ image = Image.open(abs_path).convert("RGB")
81
+ if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
82
+ setattr(model_cfg, "image_aspect_ratio", "pad")
83
+ if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
84
+ setattr(model_cfg, "mm_patch_merge_type", "flat")
85
+
86
+ image_tensor_list = process_images([image], image_processor, model_cfg)
87
+ image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
88
+ image_sizes = [image.size]
89
+ return image_tensor_list, image_sizes, None
90
+
91
+ elif media_type == "video":
92
+ if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
93
+ if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16)
94
+ if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True)
95
+
96
+ video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
97
+ image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
98
+ image_tensor = image_tensor.to(dtype=dtype, device=device)
99
+
100
+ if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
101
+ v_size = (video_frames.shape[2], video_frames.shape[1])
102
+ else:
103
+ v_size = (0, 0)
104
+
105
+ return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
106
+
107
+
108
+ def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
109
+ out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
110
+ out[-x.numel():] = x
111
+ return out
112
+
113
+
114
+ def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
115
+ if x.size(0) == L:
116
+ return x
117
+ out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
118
+ out[-x.size(0):] = x
119
+ return out
120
+
121
+
122
+ def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
123
+ p = sample.get(f"{media_type}_abs_path")
124
+ if p is None:
125
+ p = sample.get(media_type) or sample.get(f"{media_type}_path")
126
+ if p is None:
127
+ return None
128
+ if not os.path.isabs(p) and dataset_root:
129
+ p = os.path.join(dataset_root, p)
130
+ try:
131
+ return os.path.abspath(p)
132
+ except Exception:
133
+ return p
134
+
135
+
136
+ class RLVariantDatasetLLADAV(Dataset):
137
+ def __init__(self, samples: List[dict], config):
138
+ super().__init__()
139
+ self.samples = samples
140
+ self.config = config
141
+ self.variants: List[dict] = []
142
+
143
+ # logp_old_tok mapping: idx -> tensor
144
+ self.logp_old_tok: Dict[int, torch.Tensor] = {}
145
+
146
+ self._build_variants()
147
+
148
+ def _build_variants(self):
149
+ sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
150
+ question_prompt = "\nOnly give the best option."
151
+ multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
152
+
153
+ rng = np.random.default_rng(seed=self.config.training.seed or 42)
154
+ idx_counter = 0
155
+
156
+ for sid, s in enumerate(self.samples):
157
+ # Rollout produces "extracted_output" or "response", fall back if needed
158
+ response_text = s.get("response", s.get("extracted_output", ""))
159
+ if isinstance(response_text, list) and len(response_text) > 0:
160
+ response_text = response_text[0] # Just in case it's nested
161
+
162
+ # If options exist (video multiple choice), construct full prompt
163
+ if "options" in s:
164
+ prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt
165
+ else:
166
+ prompt_text = s["prompt"]
167
+
168
+ step_map = s.get("step_map", [])
169
+
170
+ if self.config.training.method == "TraceRL":
171
+ order_list = collapse_k_unique(step_map, self.config.training.shrink)
172
+ uniq_steps = sorted(set(order_list))
173
+ for st in uniq_steps:
174
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None))
175
+ idx_counter += 1
176
+
177
+ elif self.config.training.method == "random_masking":
178
+ for _ in range(self.config.training.mask_times_per_sample):
179
+ t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
180
+ mask_seed = int(rng.integers(0, 2**31 - 1))
181
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed))
182
+ idx_counter += 1
183
+
184
+ elif self.config.training.method == "coupled":
185
+ for _ in range(self.config.training.mask_times_per_sample):
186
+ t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
187
+ mask_seed = int(rng.integers(0, 2**31 - 1))
188
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed))
189
+ idx_counter += 1
190
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed))
191
+ idx_counter += 1
192
+ else:
193
+ raise ValueError(f"Unknown RL method: {self.config.training.method}")
194
+
195
+ def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed):
196
+ return dict(
197
+ idx=idx,
198
+ sample_id=sid,
199
+ prompt=prompt,
200
+ response=response,
201
+ image_abs_path=s.get("image_abs_path", None),
202
+ video_abs_path=s.get("video_abs_path", None),
203
+ reward=s.get("reward", 0.0),
204
+ var_type=var_type,
205
+ var_param=var_param,
206
+ step_map=step_map,
207
+ mask_seed=mask_seed,
208
+ )
209
+
210
+ def __len__(self):
211
+ return len(self.variants)
212
+
213
+ def __getitem__(self, idx):
214
+ return self.variants[idx]
215
+
216
+
217
+ def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
218
+ out: Dict[str, List[Any]] = {}
219
+ keys = batch[0].keys()
220
+ for k in keys:
221
+ out[k] = [b[k] for b in batch]
222
+ return out
223
+
224
+
225
+ @torch.no_grad()
226
+ def prepare_batch_noisy_embeds_rl(
227
+ accelerator,
228
+ model,
229
+ tokenizer,
230
+ image_processor,
231
+ batch: Dict[str, List[Any]],
232
+ mask_id: int,
233
+ dataset_media_root: str | None,
234
+ ) -> tuple | None:
235
+
236
+ device = accelerator.device
237
+ bsz = len(batch["idx"])
238
+
239
+ group: Dict[int, List[int]] = {}
240
+ for i, sid in enumerate(batch["sample_id"]):
241
+ group.setdefault(int(sid), []).append(i)
242
+
243
+ sample_cache: Dict[int, Dict[str, Any]] = {}
244
+ for sid, pos_list in group.items():
245
+ first_pos = pos_list[0]
246
+ prompt_text = batch["prompt"][first_pos]
247
+ response_text = batch["response"][first_pos]
248
+
249
+ sample_stub = {
250
+ "image_abs_path": batch.get("image_abs_path", [None])[first_pos],
251
+ "video_abs_path": batch.get("video_abs_path", [None])[first_pos],
252
+ }
253
+
254
+ media_type = None
255
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
256
+ if p_abs is not None:
257
+ media_type = "video"
258
+ else:
259
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
260
+ if p_abs is not None:
261
+ media_type = "image"
262
+
263
+ img_list, image_sizes, modalities = None, None, []
264
+ if p_abs is not None:
265
+ img_list, image_sizes, video_stats = load_and_process_media(
266
+ p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
267
+ )
268
+ modalities = [media_type]
269
+
270
+ if media_type == "video" and getattr(model.config, "add_time_instruction", False):
271
+ video_time, frame_time, num_frames = video_stats
272
+ time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
273
+ clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
274
+ prompt_text = f"{time_instruction}\n{clean_prompt}"
275
+
276
+ input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
277
+ end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
278
+ seq_len = len(end_header_seq)
279
+ pos_eh = -1
280
+ if seq_len > 0 and input_ids_text.numel() >= seq_len:
281
+ ids_list = input_ids_text.tolist()
282
+ for i in range(len(ids_list) - seq_len, -1, -1):
283
+ if ids_list[i:i + seq_len] == end_header_seq:
284
+ pos_eh = i + seq_len - 1
285
+ break
286
+ assert pos_eh != -1, " <|end_header_id|> not found"
287
+ start_pos = pos_eh + 1
288
+
289
+ labels_text = input_ids_text.clone()
290
+ labels_text[:start_pos] = -100
291
+
292
+ (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
293
+ input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
294
+ )
295
+ clean_embeds = clean_embeds.squeeze(0)
296
+ new_labels = new_labels.squeeze(0)
297
+
298
+ eot_id = 126348
299
+ eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
300
+ if eot_indices.numel() > 0:
301
+ last_eot_pos = eot_indices[-1, 0]
302
+ new_labels[last_eot_pos:] = -100
303
+ asst_mask = (new_labels != -100)
304
+ asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1)
305
+
306
+ sample_cache[sid] = dict(
307
+ clean_embeds=clean_embeds,
308
+ new_labels=new_labels,
309
+ asst_idx=asst_idx,
310
+ )
311
+
312
+ lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)]
313
+ Lmax = max(lengths)
314
+ mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
315
+
316
+ noisy_list, labels_list, p_mask_list, attn_list = [], [], [], []
317
+ reward_list, valid_idxs = [], []
318
+
319
+ for i in range(bsz):
320
+ sid = int(batch["sample_id"][i])
321
+ var_type = batch["var_type"][i]
322
+ var_param = batch["var_param"][i]
323
+
324
+ clean_embeds = sample_cache[sid]["clean_embeds"].clone()
325
+ new_labels = sample_cache[sid]["new_labels"].clone()
326
+ asst_idx = sample_cache[sid]["asst_idx"]
327
+
328
+ L = clean_embeds.size(0)
329
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
330
+ M = asst_idx.numel()
331
+
332
+ if M == 0:
333
+ continue # skip badly formatted inputs
334
+
335
+ if var_type == "TraceRL":
336
+ step_map = batch["step_map"][i]
337
+ order = torch.as_tensor(step_map, device=device)
338
+ m_len = min(M, order.numel())
339
+ order_full = torch.full((L,), -1, dtype=torch.long, device=device)
340
+ order_full[asst_idx[:m_len]] = order[:m_len]
341
+ step_val = int(var_param)
342
+
343
+ tgt_mask = (order_full == step_val)
344
+ pmask = tgt_mask
345
+ mask_pos = (order_full >= step_val)
346
+
347
+ noisy_embeds = clean_embeds.clone()
348
+ noisy_embeds[mask_pos] = mask_embed
349
+
350
+ elif var_type == "random_masking" or var_type.startswith("coupled"):
351
+ if M == 0:
352
+ # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward
353
+ noisy_embeds = clean_embeds.clone()
354
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
355
+ else:
356
+ t = float(var_param)
357
+ # 更安全的 seed 获取方式
358
+ mask_seed = batch.get("mask_seed", [None] * bsz)[i]
359
+ if mask_seed is None:
360
+ mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
361
+
362
+ gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
363
+ rnd_cpu = torch.rand(M, generator=gen) < t
364
+
365
+ # 如果是 coupled_comp (对比样本),需要取反
366
+ if var_type == "coupled_comp":
367
+ rnd_cpu = ~rnd_cpu
368
+
369
+ rnd = rnd_cpu.to(device)
370
+
371
+ # 🌟 上帝之手:强行兜底逻辑
372
+ # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁
373
+ if not rnd.any():
374
+ # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反,
375
+ # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。
376
+ # 这里统一下一个兜底,非常安全。
377
+ rand_pos = torch.randint(0, M, (1,), device=device)
378
+ rnd[rand_pos] = True
379
+
380
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
381
+ pmask[asst_idx] = rnd
382
+
383
+ noisy_embeds = clean_embeds.clone()
384
+ noisy_embeds[pmask] = mask_embed
385
+
386
+ # elif var_type == "random_masking" or var_type.startswith("coupled"):
387
+ # t = float(var_param)
388
+ # mask_seed = batch["mask_seed"][i]
389
+ # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
390
+ # rnd_cpu = torch.rand(M, generator=gen) < t
391
+
392
+ # if var_type == "coupled_comp":
393
+ # rnd_cpu = ~rnd_cpu
394
+
395
+ # pmask = torch.zeros(L, dtype=torch.bool, device=device)
396
+ # pmask[asst_idx] = rnd_cpu.to(device)
397
+
398
+ # noisy_embeds = clean_embeds.clone()
399
+ # noisy_embeds[pmask] = mask_embed
400
+
401
+ if not pmask.any():
402
+ print('************not pmask.any()**********')
403
+ continue
404
+
405
+ noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
406
+ labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
407
+ p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
408
+ attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
409
+ attn[-L:] = True
410
+ attn_list.append(attn)
411
+ reward_list.append(float(batch["reward"][i]))
412
+ valid_idxs.append(batch["idx"][i])
413
+
414
+ if len(noisy_list) == 0:
415
+ return None
416
+
417
+ noisy_batch = torch.stack(noisy_list, dim=0)
418
+ labels_batch = torch.stack(labels_list, dim=0)
419
+ p_mask_batch = torch.stack(p_mask_list, dim=0)
420
+ attn_batch = torch.stack(attn_list, dim=0)
421
+ reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32)
422
+
423
+ return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs
424
+
425
+
426
+ @torch.no_grad()
427
+ def compute_logp_old_tok_parallel(
428
+ accelerator,
429
+ model,
430
+ tokenizer,
431
+ image_processor,
432
+ dataset,
433
+ dl,
434
+ mask_id,
435
+ config):
436
+
437
+ model.eval()
438
+ logger.info("Computing reference log probabilities (old_lp) for all variants...")
439
+
440
+ for batch in dl:
441
+ out = prepare_batch_noisy_embeds_rl(
442
+ accelerator=accelerator,
443
+ model=model,
444
+ tokenizer=tokenizer,
445
+ image_processor=image_processor,
446
+ batch=batch,
447
+ mask_id=mask_id,
448
+ dataset_media_root=config.dataset.get("image_root", None),
449
+ )
450
+ if out is None:
451
+ continue
452
+
453
+ noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out
454
+
455
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True)
456
+ logits = model.lm_head(outputs.last_hidden_state).float()
457
+
458
+ log_probs = F.log_softmax(logits, dim=-1)
459
+ safe_labels = labels_batch.clone()
460
+ safe_labels[labels_batch == -100] = 0
461
+ tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
462
+
463
+ for i, idx in enumerate(valid_idxs):
464
+ dataset.logp_old_tok[idx] = tok_lp[i].float().cpu()
465
+
466
+ accelerator.wait_for_everyone()
467
+ model.train()
468
+
469
+
470
+ def save_checkpoint(model, tokenizer, config, accelerator, name):
471
+ output_dir = Path(config.experiment.project)
472
+ output_dir.mkdir(parents=True, exist_ok=True)
473
+
474
+ checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
475
+
476
+ if accelerator.is_main_process and checkpoints_total_limit is not None:
477
+ ckpts = sorted(
478
+ [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
479
+ key=lambda p: int(p.name.split("-")[1]),
480
+ )
481
+ if len(ckpts) >= checkpoints_total_limit:
482
+ to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
483
+ logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
484
+ for p in to_remove:
485
+ shutil.rmtree(p, ignore_errors=True)
486
+
487
+ save_base = output_dir / "ckpt"
488
+ save_base.mkdir(exist_ok=True)
489
+
490
+ model_to_save = accelerator.unwrap_model(model)
491
+ state_dict = accelerator.get_state_dict(model)
492
+
493
+ if accelerator.is_main_process:
494
+ model_to_save.save_pretrained(
495
+ save_base / name,
496
+ save_function=accelerator.save,
497
+ state_dict=state_dict,
498
+ safe_serialization=True,
499
+ )
500
+ tokenizer.save_pretrained(str(save_base / name))
501
+
502
+ metadata = {
503
+ "save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
504
+ }
505
+ with (save_base / "metadata.json").open("w") as f:
506
+ json.dump(metadata, f, indent=2)
507
+
508
+ logger.info(f"Saved model + tokenizer to {save_base / name}")
509
+
510
+
511
+ def main():
512
+ config = get_config()
513
+
514
+ project_name = config.experiment.project
515
+ if config.experiment.current_epoch == 1:
516
+ pretrained_model = config.model.pretrained_model
517
+ else:
518
+ pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name
519
+
520
+ if config.training.enable_tf32:
521
+ torch.backends.cuda.matmul.allow_tf32 = True
522
+ torch.backends.cudnn.benchmark = True
523
+ torch.backends.cudnn.deterministic = False
524
+
525
+ config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
526
+ accelerator = Accelerator(
527
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
528
+ mixed_precision=config.training.mixed_precision,
529
+ log_with='wandb',
530
+ project_dir=config.experiment.logging_dir,
531
+ split_batches=True,
532
+ )
533
+
534
+ logging.basicConfig(
535
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
536
+ datefmt="%m/%d/%Y %H:%M:%S",
537
+ level=logging.INFO,
538
+ )
539
+ logger.info(accelerator.state, main_process_only=False)
540
+ if accelerator.is_local_main_process:
541
+ set_verbosity_info()
542
+ else:
543
+ set_verbosity_error()
544
+
545
+ if accelerator.is_main_process:
546
+ resume_wandb_run = config.wandb.resume
547
+ run_id = config.wandb.get("run_id", None)
548
+ if run_id is None:
549
+ resume_wandb_run = False
550
+ run_id = wandb.util.generate_id()
551
+ config.wandb.run_id = run_id
552
+
553
+ wandb_init_kwargs = dict(
554
+ name=config.experiment.project,
555
+ id=run_id,
556
+ resume=resume_wandb_run,
557
+ entity=config.wandb.get("entity", None),
558
+ config_exclude_keys=[],
559
+ )
560
+ wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
561
+ wandb_config.pop("experiment.resume_from_checkpoint", None)
562
+
563
+ accelerator.init_trackers(
564
+ config.experiment.project,
565
+ config=wandb_config,
566
+ init_kwargs={"wandb": wandb_init_kwargs},
567
+ )
568
+
569
+ if accelerator.is_main_process:
570
+ os.makedirs(config.experiment.project, exist_ok=True)
571
+ config_path = Path(config.experiment.project) / "config.yaml"
572
+ logging.info(f"Saving config to {config_path}")
573
+ OmegaConf.save(config, config_path)
574
+
575
+ if config.training.seed is not None:
576
+ set_seed(config.training.seed)
577
+
578
+ logger.info("Loading LLaDA-V model and tokenizer")
579
+
580
+ tokenizer, model, image_processor, _ = load_pretrained_model(
581
+ pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
582
+ )
583
+ model = model.to(accelerator.device)
584
+
585
+ special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
586
+ num_new = tokenizer.add_special_tokens(special_tokens)
587
+ if num_new > 0:
588
+ model.resize_token_embeddings(len(tokenizer))
589
+ if tokenizer.pad_token is None:
590
+ tokenizer.pad_token = tokenizer.eos_token
591
+
592
+ if getattr(config.training, "gradient_checkpointing_enable", False):
593
+ try:
594
+ model.enable_input_require_grads()
595
+ except Exception:
596
+ pass
597
+ try:
598
+ model.gradient_checkpointing_enable()
599
+ except Exception:
600
+ pass
601
+
602
+ mask_id = 126336
603
+
604
+ optimizer_config = config.optimizer.params
605
+ no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
606
+ optimizer_grouped_parameters = [
607
+ {
608
+ "params": [p for n, p in model.named_parameters() if
609
+ p.requires_grad and not any(nd in n for nd in no_decay)],
610
+ "weight_decay": optimizer_config.weight_decay,
611
+ },
612
+ {
613
+ "params": [p for n, p in model.named_parameters() if
614
+ p.requires_grad and any(nd in n for nd in no_decay)],
615
+ "weight_decay": 0.0,
616
+ },
617
+ ]
618
+
619
+ optimizer_type = config.optimizer.name
620
+ if optimizer_type == "adamw":
621
+ optimizer = AdamW(
622
+ optimizer_grouped_parameters,
623
+ lr=optimizer_config.learning_rate,
624
+ betas=(optimizer_config.beta1, optimizer_config.beta2),
625
+ weight_decay=optimizer_config.weight_decay,
626
+ eps=optimizer_config.epsilon,
627
+ )
628
+ else:
629
+ raise ValueError(f"Optimizer {optimizer_type} not supported")
630
+
631
+
632
+ logger.info("Preprocessing Data")
633
+
634
+ with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f:
635
+ dataset_load = json.load(f)
636
+
637
+ dataset_lm = RLVariantDatasetLLADAV(dataset_load, config)
638
+
639
+ logger.info("Preparing model, optimizer and dataloaders")
640
+
641
+ total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
642
+ num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm)
643
+ num_train_epochs = config.training.num_train_epochs
644
+ max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
645
+
646
+ lr_scheduler = get_scheduler(
647
+ config.lr_scheduler.scheduler,
648
+ optimizer=optimizer,
649
+ num_training_steps=max_train_steps,
650
+ num_warmup_steps=config.lr_scheduler.params.warmup_steps,
651
+ min_lr_scale=config.lr_scheduler.params.min_lr_scale
652
+ )
653
+
654
+ train_dataloader_lm = DataLoader(
655
+ dataset_lm,
656
+ batch_size=config.training.batch_size_lm,
657
+ sampler=None,
658
+ collate_fn=collate_variants,
659
+ num_workers=0,
660
+ shuffle=False
661
+ )
662
+
663
+ model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare(
664
+ model, optimizer, lr_scheduler, train_dataloader_lm
665
+ )
666
+
667
+ logger.info("***** Running inference *****")
668
+ compute_logp_old_tok_parallel(
669
+ accelerator,
670
+ model,
671
+ tokenizer,
672
+ image_processor,
673
+ dataset_lm,
674
+ train_dataloader_lm,
675
+ mask_id=mask_id,
676
+ config=config
677
+ )
678
+
679
+ logger.info("***** Running training *****")
680
+ logger.info(f" Num variants = {len(dataset_lm)}")
681
+ logger.info(f" Num training steps = {max_train_steps}")
682
+ logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
683
+ logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
684
+ logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
685
+
686
+ first_epoch = 0
687
+ data_time_m = AverageMeter()
688
+ end = time.time()
689
+ grad_accum = config.training.gradient_accumulation_steps
690
+
691
+ def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok):
692
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
693
+ logits = model.lm_head(outputs.last_hidden_state).float()
694
+
695
+ B, T, V = logits.shape
696
+ adv = torch.as_tensor(adv, device=noisy_embeds.device).detach()
697
+
698
+ log_probs = F.log_softmax(logits, dim=-1)
699
+ safe_labels = labels.clone()
700
+ safe_labels[labels == -100] = 0
701
+ logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
702
+
703
+ ratio = torch.exp(logp_new_tok - logp_old_tok)
704
+ clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps)
705
+ adv_tok = adv.unsqueeze(1)
706
+
707
+ surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok)
708
+ surrogate_tok = surrogate_tok * p_mask_lm
709
+ num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1)
710
+ surrogate_tok = surrogate_tok.sum(dim=1) / num_mask
711
+ policy_loss = - (surrogate_tok.sum() / B)
712
+
713
+ # KL penalty
714
+ kl_loss = torch.tensor(0.0, device=policy_loss.device)
715
+ if config.training.beta > 0:
716
+ kl_seq = logp_new_tok - logp_old_tok
717
+ if config.training.use_kl_estimator_k3:
718
+ kl_seq = (-kl_seq).exp() - 1.0 + kl_seq
719
+ kl_seq = (kl_seq * p_mask_lm).sum(dim=1)
720
+ kl_loss = config.training.beta * kl_seq.sum() / B
721
+ total_loss = policy_loss + kl_loss
722
+ else:
723
+ total_loss = policy_loss
724
+
725
+ return total_loss
726
+
727
+ from tqdm.auto import tqdm
728
+
729
+ cprint("start rl training...", "red")
730
+ for epoch in range(first_epoch, num_train_epochs):
731
+ model.train()
732
+ progress_bar = tqdm(
733
+ train_dataloader_lm,
734
+ desc=f"Epoch {epoch+1}/{num_train_epochs}",
735
+ disable=not accelerator.is_local_main_process,
736
+ dynamic_ncols=True,
737
+ leave=True
738
+ )
739
+
740
+ step = 0
741
+ for step, batch in enumerate(progress_bar, start=1):
742
+ data_time_m.update(time.time() - end)
743
+
744
+ out = prepare_batch_noisy_embeds_rl(
745
+ accelerator=accelerator,
746
+ model=model,
747
+ tokenizer=tokenizer,
748
+ image_processor=image_processor,
749
+ batch=batch,
750
+ mask_id=mask_id,
751
+ dataset_media_root=config.dataset.get("image_root", None),
752
+ )
753
+
754
+ if out is None:
755
+ continue
756
+
757
+ noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out
758
+
759
+ # Align old log probabilities shape with current noisy_embeds
760
+ L_current = noisy_embeds.shape[1]
761
+ old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device)
762
+ for i, idx in enumerate(valid_idxs):
763
+ old_t = dataset_lm.logp_old_tok[idx]
764
+ l_min = min(L_current, old_t.shape[0])
765
+ old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device)
766
+
767
+ loss_lm = forward_process(
768
+ noisy_embeds=noisy_embeds,
769
+ labels=labels,
770
+ p_mask_lm=p_mask_lm,
771
+ attn_mask=attn_mask,
772
+ adv=rewards,
773
+ logp_old_tok=old_lp_batch
774
+ )
775
+
776
+ loss_lm = loss_lm / grad_accum
777
+ accelerator.backward(loss_lm)
778
+
779
+ if step % grad_accum == 0:
780
+ accelerator.log({
781
+ "train/total_loss": loss_lm.item() * grad_accum,
782
+ "train/policy_loss": p_loss.item(),
783
+ "train/kl_loss": k_loss.item(),
784
+ "train/avg_ratio": avg_ratio.item(),
785
+ "train/learning_rate": current_lr,
786
+ "train/epoch": epoch,
787
+ })
788
+
789
+ if config.training.max_grad_norm is not None:
790
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
791
+ optimizer.step()
792
+ lr_scheduler.step()
793
+ optimizer.zero_grad(set_to_none=True)
794
+ torch.cuda.empty_cache()
795
+
796
+ # Catch remaining gradients
797
+ if step % grad_accum != 0:
798
+ if config.training.max_grad_norm is not None:
799
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
800
+ optimizer.step()
801
+ lr_scheduler.step()
802
+ optimizer.zero_grad(set_to_none=True)
803
+ torch.cuda.empty_cache()
804
+
805
+ accelerator.wait_for_everyone()
806
+
807
+ save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
808
+ if config.experiment.current_epoch % config.experiment.save_every == 0:
809
+ save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}")
810
+
811
+ accelerator.end_training()
812
+
813
+
814
+ if __name__ == "__main__":
815
+ main()
wandb/run-20260324_164425-eb44q2tb/files/config.yaml ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.25.0
4
+ code_path: code/train/rl_lladav.py
5
+ e:
6
+ sr0a7mkap2eiuw8cpbm7o5idgj2l6pki:
7
+ args:
8
+ - config=configs/rl_lladav_video_reversal.yaml
9
+ - experiment.current_epoch=15
10
+ codePath: train/rl_lladav.py
11
+ codePathLocal: train/rl_lladav.py
12
+ cpu_count: 96
13
+ cpu_count_logical: 192
14
+ cudaVersion: "12.4"
15
+ disk:
16
+ /:
17
+ total: "806578724864"
18
+ used: "572337045504"
19
+ email: shixi_runqing@staff.weibo.com
20
+ executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10
21
+ git:
22
+ commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
23
+ remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
24
+ gpu: NVIDIA H800
25
+ gpu_count: 8
26
+ gpu_nvidia:
27
+ - architecture: Hopper
28
+ cudaCores: 16896
29
+ memoryTotal: "85520809984"
30
+ name: NVIDIA H800
31
+ uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
32
+ - architecture: Hopper
33
+ cudaCores: 16896
34
+ memoryTotal: "85520809984"
35
+ name: NVIDIA H800
36
+ uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
37
+ - architecture: Hopper
38
+ cudaCores: 16896
39
+ memoryTotal: "85520809984"
40
+ name: NVIDIA H800
41
+ uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
42
+ - architecture: Hopper
43
+ cudaCores: 16896
44
+ memoryTotal: "85520809984"
45
+ name: NVIDIA H800
46
+ uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
47
+ - architecture: Hopper
48
+ cudaCores: 16896
49
+ memoryTotal: "85520809984"
50
+ name: NVIDIA H800
51
+ uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
52
+ - architecture: Hopper
53
+ cudaCores: 16896
54
+ memoryTotal: "85520809984"
55
+ name: NVIDIA H800
56
+ uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
57
+ - architecture: Hopper
58
+ cudaCores: 16896
59
+ memoryTotal: "85520809984"
60
+ name: NVIDIA H800
61
+ uuid: GPU-2775489f-18c8-2328-8829-c46238575379
62
+ - architecture: Hopper
63
+ cudaCores: 16896
64
+ memoryTotal: "85520809984"
65
+ name: NVIDIA H800
66
+ uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
67
+ host: b1301b38c75f
68
+ memory:
69
+ total: "2164113813504"
70
+ os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
71
+ program: /data4/runqing/dLLM-RL/train/rl_lladav.py
72
+ python: CPython 3.10.19
73
+ root: /data4/runqing/dLLM-RL
74
+ startedAt: "2026-03-24T08:44:25.714950Z"
75
+ writerId: sr0a7mkap2eiuw8cpbm7o5idgj2l6pki
76
+ m: []
77
+ python_version: 3.10.19
78
+ t:
79
+ "1":
80
+ - 1
81
+ - 5
82
+ - 11
83
+ - 41
84
+ - 49
85
+ - 53
86
+ - 71
87
+ - 98
88
+ "2":
89
+ - 1
90
+ - 5
91
+ - 11
92
+ - 41
93
+ - 49
94
+ - 51
95
+ - 53
96
+ - 71
97
+ - 98
98
+ "3":
99
+ - 13
100
+ - 14
101
+ "4": 3.10.19
102
+ "5": 0.25.0
103
+ "6": 4.52.4
104
+ "12": 0.25.0
105
+ "13": linux-x86_64
106
+ config:
107
+ value: configs/rl_lladav_video_reversal.yaml
108
+ dataset.data_type:
109
+ value: video_reverse
110
+ dataset.optimization_data:
111
+ value: rl_data
112
+ dataset.train_dataset:
113
+ value: video_reversal_rl_train
114
+ evaluation.batch_size:
115
+ value: 1
116
+ evaluation.block_size:
117
+ value: 32
118
+ evaluation.data_type:
119
+ value: video_reverse
120
+ evaluation.dynamic_threshold:
121
+ value: 0.9
122
+ evaluation.eval_dataset:
123
+ value: video_reversal_rl_eval
124
+ evaluation.further_horizon:
125
+ value: 128
126
+ evaluation.if_eval:
127
+ value: true
128
+ evaluation.max_gen_length:
129
+ value: 128
130
+ evaluation.num_response_per_task:
131
+ value: 3
132
+ evaluation.remasking_strategy.0:
133
+ value: low_confidence_static
134
+ evaluation.remasking_strategy.1:
135
+ value: low_confidence_dynamic
136
+ evaluation.steps:
137
+ value: 128
138
+ evaluation.target:
139
+ value: confidence
140
+ evaluation.temperature:
141
+ value: 0.1
142
+ evaluation.use_cache:
143
+ value: true
144
+ execute.num_chunk:
145
+ value: 128
146
+ experiment.current_epoch:
147
+ value: 15
148
+ experiment.deepspeed_file:
149
+ value: 1_node_8_gpus_deepspeed_zero2_rl
150
+ experiment.eval_every:
151
+ value: 5
152
+ experiment.function:
153
+ value: train
154
+ experiment.logging_dir:
155
+ value: rl_lladav_video_reversal/logs
156
+ experiment.node_index:
157
+ value: 0
158
+ experiment.num_node:
159
+ value: 1
160
+ experiment.project:
161
+ value: rl_lladav_video_reversal
162
+ experiment.save_every:
163
+ value: 10
164
+ experiment.start_from_scratch:
165
+ value: true
166
+ experiment.total_step:
167
+ value: 20
168
+ lr_scheduler.params.learning_rate:
169
+ value: 1e-06
170
+ lr_scheduler.params.min_lr_scale:
171
+ value: 1
172
+ lr_scheduler.params.warmup_steps:
173
+ value: 0
174
+ lr_scheduler.scheduler:
175
+ value: cosine
176
+ model.image_resolution:
177
+ value: 512
178
+ model.model_base:
179
+ value: lladav
180
+ model.optimized_name:
181
+ value: optimized
182
+ model.pretrained_model:
183
+ value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
184
+ model.vq_model_path:
185
+ value: /data4/runqing/hf/magvitv2
186
+ optimizer.name:
187
+ value: adamw
188
+ optimizer.params.beta1:
189
+ value: 0.9
190
+ optimizer.params.beta2:
191
+ value: 0.999
192
+ optimizer.params.epsilon:
193
+ value: 1e-08
194
+ optimizer.params.learning_rate:
195
+ value: 1e-06
196
+ optimizer.params.scale_lr:
197
+ value: false
198
+ optimizer.params.weight_decay:
199
+ value: 0
200
+ reward.answer_must_in_box:
201
+ value: true
202
+ reward.strict_len_check:
203
+ value: false
204
+ rollout.batch_size:
205
+ value: 1
206
+ rollout.block_size:
207
+ value: 32
208
+ rollout.dynamic_threshold:
209
+ value: 0.9
210
+ rollout.further_horizon:
211
+ value: 128
212
+ rollout.max_gen_length:
213
+ value: 128
214
+ rollout.num_response_per_task:
215
+ value: 8
216
+ rollout.num_task_per_step:
217
+ value: 128
218
+ rollout.remasking_strategy:
219
+ value: low_confidence_dynamic
220
+ rollout.steps:
221
+ value: 128
222
+ rollout.target:
223
+ value: confidence
224
+ rollout.temperature:
225
+ value: 0.8
226
+ rollout.use_cache:
227
+ value: true
228
+ system.HF_HOME:
229
+ value: null
230
+ system.HTTP_PROXY:
231
+ value: null
232
+ system.base_dir:
233
+ value: /data4/runqing/dLLM-RL
234
+ system.env_name:
235
+ value: /data8/runqing/miniconda/envs/dllm-rl
236
+ system.envs_dir:
237
+ value: /data8/runqing/miniconda/envs/dllm-rl
238
+ training.batch_size_lm:
239
+ value: 1
240
+ training.beta:
241
+ value: 0.01
242
+ training.enable_tf32:
243
+ value: true
244
+ training.eps:
245
+ value: 0.2
246
+ training.gradient_accumulation_steps:
247
+ value: 64
248
+ training.lower_p:
249
+ value: 0.35
250
+ training.mask_times_per_sample:
251
+ value: 2
252
+ training.max_gen_length:
253
+ value: 128
254
+ training.max_grad_norm:
255
+ value: 1
256
+ training.max_prompt_len:
257
+ value: 512
258
+ training.method:
259
+ value: random_masking
260
+ training.mixed_precision:
261
+ value: bf16
262
+ training.num_train_epochs:
263
+ value: 1
264
+ training.post_num:
265
+ value: 1
266
+ training.seed:
267
+ value: 10086
268
+ training.shrink:
269
+ value: 8
270
+ training.upper_p:
271
+ value: 0.9
272
+ training.use_kl_estimator_k3:
273
+ value: true
274
+ wandb.entity:
275
+ value: null
276
+ wandb.resume:
277
+ value: auto
278
+ wandb.run_id:
279
+ value: eb44q2tb
wandb/run-20260324_164425-eb44q2tb/files/diff.patch ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..9f94fa7 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,87 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [
158
+ + {
159
+ + "name": "stdout",
160
+ + "output_type": "stream",
161
+ + "text": [
162
+ + "200\n"
163
+ + ]
164
+ + },
165
+ + {
166
+ + "ename": "",
167
+ + "evalue": "",
168
+ + "output_type": "error",
169
+ + "traceback": [
170
+ + "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
171
+ + "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
172
+ + "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
173
+ + "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
174
+ + ]
175
+ + }
176
+ + ],
177
+ + "source": [
178
+ + "import json\n",
179
+ + "import random\n",
180
+ + "random.seed(2026)\n",
181
+ + "\n",
182
+ + "\n",
183
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
184
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
185
+ + "save_data_list = []\n",
186
+ + "save_num = 0\n",
187
+ + "with open(data_path, 'r') as fin, \\\n",
188
+ + " open(save_path, 'w') as fout:\n",
189
+ + " raw_data = json.load(fin)\n",
190
+ + "\n",
191
+ + " # random.shuffle(raw_data)\n",
192
+ + " for did, data in enumerate(raw_data):\n",
193
+ + " # if did > 1999:\n",
194
+ + " if save_num < 200:\n",
195
+ + " save_data_list.append(data)\n",
196
+ + " save_num += 1\n",
197
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
198
+ + " print(len(save_data_list))\n"
199
+ + ]
200
+ }
201
+ ],
202
+ "metadata": {
203
+ @@ -85,7 +166,7 @@
204
+ "name": "python",
205
+ "nbconvert_exporter": "python",
206
+ "pygments_lexer": "ipython3",
207
+ - "version": "3.10.18"
208
+ + "version": "3.10.19"
209
+ }
210
+ },
211
+ "nbformat": 4,
212
+ diff --git a/rl_v.py b/rl_v.py
213
+ index 8f2c0f1..79d2e5b 100644
214
+ --- a/rl_v.py
215
+ +++ b/rl_v.py
216
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
217
+ def env_prefix(env) -> str:
218
+ return (
219
+ "source ~/.bashrc && "
220
+ - f"source activate {env} && "
221
+ + "source /data8/runqing/miniconda/bin/activate && "
222
+ + "conda init && "
223
+ + f"conda activate {env} && "
224
+ )
225
+
226
+ config = get_config()
227
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
228
+ index 4068fc8..d284a23 100644
229
+ --- a/sample/llava/conversation.py
230
+ +++ b/sample/llava/conversation.py
231
+ @@ -99,7 +99,7 @@ class Conversation:
232
+ if self.version == "llama_v3":
233
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
234
+ elif self.version == "llava_llada":
235
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
236
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
237
+ else:
238
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
239
+ chat_template_messages = [{"role": "system", "content": self.system}]
240
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
241
+ index 3db5302..96e33b5 100644
242
+ --- a/sample/llava/model/builder.py
243
+ +++ b/sample/llava/model/builder.py
244
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
245
+ elif "llada" in model_name.lower():
246
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
247
+
248
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
249
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
250
+ +
251
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
252
+ if customized_config is None:
253
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
254
+ else:
255
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
256
+ index 4068fc8..d284a23 100644
257
+ --- a/train/llava/conversation.py
258
+ +++ b/train/llava/conversation.py
259
+ @@ -99,7 +99,7 @@ class Conversation:
260
+ if self.version == "llama_v3":
261
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
262
+ elif self.version == "llava_llada":
263
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
264
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
265
+ else:
266
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
267
+ chat_template_messages = [{"role": "system", "content": self.system}]
268
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
269
+ index 3db5302..aafab95 100644
270
+ --- a/train/llava/model/builder.py
271
+ +++ b/train/llava/model/builder.py
272
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
273
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
274
+ elif "llada" in model_name.lower():
275
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
276
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
277
+
278
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
279
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
280
+ if customized_config is None:
281
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
282
+ else:
283
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
284
+ index 3da28dc..8b17da3 100644
285
+ --- a/train/llava/model/language_model/llava_llada.py
286
+ +++ b/train/llava/model/language_model/llava_llada.py
287
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
288
+ dpo_forward: Optional[bool] = None,
289
+ cache_position=None,
290
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
291
+ -
292
+ + conversation_ids = None
293
+ if inputs_embeds is None and attention_mask is not None:
294
+ # donate multi-dialogue
295
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
296
+ diff --git a/train/rl_llada.py b/train/rl_llada.py
297
+ index 916ce23..00693be 100644
298
+ --- a/train/rl_llada.py
299
+ +++ b/train/rl_llada.py
300
+ @@ -96,7 +96,7 @@ def main():
301
+ accelerator = Accelerator(
302
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
303
+ mixed_precision=config.training.mixed_precision,
304
+ - log_with=None,
305
+ + log_with='wandb',
306
+ project_dir=config.experiment.logging_dir,
307
+ split_batches=True,
308
+ )
wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..9f94fa7 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,87 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [
158
+ + {
159
+ + "name": "stdout",
160
+ + "output_type": "stream",
161
+ + "text": [
162
+ + "200\n"
163
+ + ]
164
+ + },
165
+ + {
166
+ + "ename": "",
167
+ + "evalue": "",
168
+ + "output_type": "error",
169
+ + "traceback": [
170
+ + "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
171
+ + "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
172
+ + "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
173
+ + "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
174
+ + ]
175
+ + }
176
+ + ],
177
+ + "source": [
178
+ + "import json\n",
179
+ + "import random\n",
180
+ + "random.seed(2026)\n",
181
+ + "\n",
182
+ + "\n",
183
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
184
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
185
+ + "save_data_list = []\n",
186
+ + "save_num = 0\n",
187
+ + "with open(data_path, 'r') as fin, \\\n",
188
+ + " open(save_path, 'w') as fout:\n",
189
+ + " raw_data = json.load(fin)\n",
190
+ + "\n",
191
+ + " # random.shuffle(raw_data)\n",
192
+ + " for did, data in enumerate(raw_data):\n",
193
+ + " # if did > 1999:\n",
194
+ + " if save_num < 200:\n",
195
+ + " save_data_list.append(data)\n",
196
+ + " save_num += 1\n",
197
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
198
+ + " print(len(save_data_list))\n"
199
+ + ]
200
+ }
201
+ ],
202
+ "metadata": {
203
+ @@ -85,7 +166,7 @@
204
+ "name": "python",
205
+ "nbconvert_exporter": "python",
206
+ "pygments_lexer": "ipython3",
207
+ - "version": "3.10.18"
208
+ + "version": "3.10.19"
209
+ }
210
+ },
211
+ "nbformat": 4,
212
+ diff --git a/rl_v.py b/rl_v.py
213
+ index 8f2c0f1..79d2e5b 100644
214
+ --- a/rl_v.py
215
+ +++ b/rl_v.py
216
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
217
+ def env_prefix(env) -> str:
218
+ return (
219
+ "source ~/.bashrc && "
220
+ - f"source activate {env} && "
221
+ + "source /data8/runqing/miniconda/bin/activate && "
222
+ + "conda init && "
223
+ + f"conda activate {env} && "
224
+ )
225
+
226
+ config = get_config()
227
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
228
+ index 4068fc8..d284a23 100644
229
+ --- a/sample/llava/conversation.py
230
+ +++ b/sample/llava/conversation.py
231
+ @@ -99,7 +99,7 @@ class Conversation:
232
+ if self.version == "llama_v3":
233
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
234
+ elif self.version == "llava_llada":
235
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
236
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
237
+ else:
238
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
239
+ chat_template_messages = [{"role": "system", "content": self.system}]
240
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
241
+ index 3db5302..96e33b5 100644
242
+ --- a/sample/llava/model/builder.py
243
+ +++ b/sample/llava/model/builder.py
244
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
245
+ elif "llada" in model_name.lower():
246
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
247
+
248
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
249
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
250
+ +
251
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
252
+ if customized_config is None:
253
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
254
+ else:
255
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
256
+ index 4068fc8..d284a23 100644
257
+ --- a/train/llava/conversation.py
258
+ +++ b/train/llava/conversation.py
259
+ @@ -99,7 +99,7 @@ class Conversation:
260
+ if self.version == "llama_v3":
261
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
262
+ elif self.version == "llava_llada":
263
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
264
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
265
+ else:
266
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
267
+ chat_template_messages = [{"role": "system", "content": self.system}]
268
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
269
+ index 3db5302..aafab95 100644
270
+ --- a/train/llava/model/builder.py
271
+ +++ b/train/llava/model/builder.py
272
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
273
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
274
+ elif "llada" in model_name.lower():
275
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
276
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
277
+
278
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
279
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
280
+ if customized_config is None:
281
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
282
+ else:
283
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
284
+ index 3da28dc..8b17da3 100644
285
+ --- a/train/llava/model/language_model/llava_llada.py
286
+ +++ b/train/llava/model/language_model/llava_llada.py
287
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
288
+ dpo_forward: Optional[bool] = None,
289
+ cache_position=None,
290
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
291
+ -
292
+ + conversation_ids = None
293
+ if inputs_embeds is None and attention_mask is not None:
294
+ # donate multi-dialogue
295
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
296
+ diff --git a/train/rl_llada.py b/train/rl_llada.py
297
+ index 916ce23..00693be 100644
298
+ --- a/train/rl_llada.py
299
+ +++ b/train/rl_llada.py
300
+ @@ -96,7 +96,7 @@ def main():
301
+ accelerator = Accelerator(
302
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
303
+ mixed_precision=config.training.mixed_precision,
304
+ - log_with=None,
305
+ + log_with='wandb',
306
+ project_dir=config.experiment.logging_dir,
307
+ split_batches=True,
308
+ )
wandb/run-20260324_164425-eb44q2tb/files/output.log ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 03/24/2026 16:44:30 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml
2
+ 03/24/2026 16:44:30 - INFO - __main__ - Loading LLaDA-V model and tokenizer
3
+ Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal/ckpt/optimized
4
+ Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
5
+ Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.39s/it]
6
+ Rank 0: Model Class: LlavaLLaDAModelLM
7
+ 03/24/2026 16:44:52 - INFO - __main__ - Preprocessing Data
8
+ 03/24/2026 16:44:52 - INFO - __main__ - Preparing model, optimizer and dataloaders
9
+ /data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
10
+ If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
11
+ warnings.warn(
12
+ Before initializing optimizer states
13
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
14
+ CPU Virtual Memory: used = 178.31 GB, percent = 8.8%
15
+ After initializing optimizer states
16
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
17
+ CPU Virtual Memory: used = 200.78 GB, percent = 10.0%
18
+ After initializing ZeRO optimizer
19
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
20
+ CPU Virtual Memory: used = 202.31 GB, percent = 10.0%
21
+ 03/24/2026 16:46:09 - INFO - __main__ - ***** Running inference *****
22
+ 03/24/2026 16:46:09 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants...
23
+ 03/24/2026 16:50:40 - INFO - __main__ - ***** Running training *****
24
+ 03/24/2026 16:50:40 - INFO - __main__ - Num variants = 704
25
+ 03/24/2026 16:50:40 - INFO - __main__ - Num training steps = 3
26
+ 03/24/2026 16:50:40 - INFO - __main__ - Instantaneous batch size per device = 1
27
+ 03/24/2026 16:50:40 - INFO - __main__ - Total train batch size (parallel * accum) = 512
28
+ 03/24/2026 16:50:40 - INFO - __main__ - Gradient Accumulation steps = 64
29
+ start rl training...
30
+ Epoch 1/1: 72%|███████▏ | 63/88 [11:05<04:24, 10.56s/it]
31
+ Traceback (most recent call last):
32
+ File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 815, in <module>
33
+ main()
34
+ File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 782, in main
35
+ "train/policy_loss": p_loss.item(),
36
+ NameError: name 'p_loss' is not defined
37
+ [rank0]: Traceback (most recent call last):
38
+ [rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 815, in <module>
39
+ [rank0]: main()
40
+ [rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 782, in main
41
+ [rank0]: "train/policy_loss": p_loss.item(),
42
+ [rank0]: NameError: name 'p_loss' is not defined
wandb/run-20260324_164425-eb44q2tb/files/requirements.txt ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ transformers==4.52.4
2
+ nvidia-nvtx-cu12==12.4.127
3
+ deepspeed==0.18.6
4
+ typing-inspection==0.4.2
5
+ scipy==1.15.3
6
+ MarkupSafe==3.0.3
7
+ nvidia-cuda-runtime-cu12==12.4.127
8
+ yarl==1.22.0
9
+ tornado==6.5.4
10
+ nvidia-curand-cu12==10.3.5.147
11
+ nest-asyncio==1.6.0
12
+ safetensors==0.7.0
13
+ async-timeout==5.0.1
14
+ typing_extensions==4.15.0
15
+ asttokens==3.0.1
16
+ cuda-python==13.1.1
17
+ executing==2.2.1
18
+ nvidia-cudnn-frontend==1.18.0
19
+ comm==0.2.3
20
+ tokenizers==0.21.4
21
+ six==1.17.0
22
+ Pygments==2.19.2
23
+ nvidia-cusparselt-cu12==0.6.2
24
+ apache-tvm-ffi==0.1.8.post2
25
+ flash_attn==2.7.4.post1
26
+ tqdm==4.67.3
27
+ tensorboard-data-server==0.7.2
28
+ annotated-types==0.7.0
29
+ math-verify==0.9.0
30
+ idna==3.11
31
+ regex==2026.1.15
32
+ nvidia-cutlass-dsl-libs-base==4.4.0
33
+ flashinfer-python==0.6.3
34
+ click==8.3.1
35
+ pytz==2025.2
36
+ wheel==0.46.3
37
+ ipykernel==7.2.0
38
+ stack-data==0.6.3
39
+ absl-py==2.4.0
40
+ wandb==0.25.0
41
+ pandas==2.3.3
42
+ termcolor==3.3.0
43
+ sympy==1.13.1
44
+ joblib==1.5.3
45
+ prompt_toolkit==3.0.52
46
+ attrs==25.4.0
47
+ nvidia-cusolver-cu12==11.6.1.9
48
+ xxhash==3.6.0
49
+ aiohappyeyeballs==2.6.1
50
+ python-dateutil==2.9.0.post0
51
+ frozenlist==1.8.0
52
+ nvidia-ml-py==13.590.48
53
+ multiprocess==0.70.18
54
+ decorator==5.2.1
55
+ cuda-pathfinder==1.3.4
56
+ multidict==6.7.1
57
+ pure_eval==0.2.3
58
+ nvidia-cufft-cu12==11.2.1.3
59
+ omegaconf==2.3.0
60
+ PyYAML==6.0.3
61
+ debugpy==1.8.20
62
+ psutil==7.2.2
63
+ ninja==1.13.0
64
+ nvidia-cublas-cu12==12.4.5.8
65
+ triton==3.2.0
66
+ antlr4-python3-runtime==4.9.3
67
+ peft==0.18.1
68
+ aiosignal==1.4.0
69
+ accelerate==1.12.0
70
+ liger_kernel==0.7.0
71
+ anyio==4.12.1
72
+ decord==0.6.0
73
+ h11==0.16.0
74
+ urllib3==2.6.3
75
+ wcwidth==0.6.0
76
+ networkx==3.4.2
77
+ huggingface_hub==0.36.2
78
+ tzdata==2025.3
79
+ xformers==0.0.29.post3
80
+ tabulate==0.9.0
81
+ certifi==2026.1.4
82
+ tensorboard==2.20.0
83
+ matplotlib-inline==0.2.1
84
+ pip==26.0.1
85
+ smmap==5.0.2
86
+ nvidia-cuda-cupti-cu12==12.4.127
87
+ Markdown==3.10.2
88
+ nvidia-nccl-cu12==2.21.5
89
+ aiohttp==3.13.3
90
+ scikit-learn==1.7.2
91
+ nvidia-cutlass-dsl==4.4.0
92
+ setuptools==82.0.0
93
+ jupyter_core==5.9.1
94
+ asyncio==4.0.0
95
+ einops==0.8.2
96
+ pydantic==2.12.5
97
+ av==16.1.0
98
+ protobuf==6.33.5
99
+ evaluate==0.4.6
100
+ Jinja2==3.1.6
101
+ exceptiongroup==1.3.1
102
+ ipython==8.38.0
103
+ sentry-sdk==2.53.0
104
+ httpx==0.28.1
105
+ msgpack==1.1.2
106
+ platformdirs==4.9.2
107
+ hjson==3.1.0
108
+ torchvision==0.21.0
109
+ parso==0.8.6
110
+ cuda-bindings==13.1.1
111
+ pexpect==4.9.0
112
+ threadpoolctl==3.6.0
113
+ pyarrow==23.0.1
114
+ jedi==0.19.2
115
+ nvidia-cuda-nvrtc-cu12==12.4.127
116
+ Werkzeug==3.1.5
117
+ fsspec==2025.10.0
118
+ pydantic_core==2.41.5
119
+ GitPython==3.1.46
120
+ propcache==0.4.1
121
+ mpmath==1.3.0
122
+ nvidia-nvjitlink-cu12==12.4.127
123
+ nvidia-cusparse-cu12==12.3.1.170
124
+ hf-xet==1.2.0
125
+ charset-normalizer==3.4.4
126
+ dill==0.4.0
127
+ datasets==4.5.0
128
+ py-cpuinfo==9.0.0
129
+ sentencepiece==0.2.1
130
+ latex2sympy2_extended==1.11.0
131
+ jupyter_client==8.8.0
132
+ pyzmq==27.1.0
133
+ numpy==2.2.6
134
+ gitdb==4.0.12
135
+ grpcio==1.78.0
136
+ packaging==26.0
137
+ requests==2.32.5
138
+ nvidia-cudnn-cu12==9.1.0.70
139
+ httpcore==1.0.9
140
+ traitlets==5.14.3
141
+ filelock==3.24.2
142
+ ptyprocess==0.7.0
143
+ pillow==12.1.1
144
+ torch==2.6.0
145
+ more-itertools==10.8.0
146
+ wheel==0.46.3
147
+ jaraco.text==4.0.0
148
+ zipp==3.23.0
149
+ autocommand==2.2.2
150
+ jaraco.context==6.1.0
151
+ jaraco.functools==4.4.0
152
+ tomli==2.4.0
153
+ backports.tarfile==1.2.0
154
+ importlib_metadata==8.7.1
155
+ platformdirs==4.4.0
156
+ packaging==26.0
wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.19",
4
+ "startedAt": "2026-03-24T08:44:25.714950Z",
5
+ "args": [
6
+ "config=configs/rl_lladav_video_reversal.yaml",
7
+ "experiment.current_epoch=15"
8
+ ],
9
+ "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py",
10
+ "codePath": "train/rl_lladav.py",
11
+ "codePathLocal": "train/rl_lladav.py",
12
+ "git": {
13
+ "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
14
+ "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
15
+ },
16
+ "email": "shixi_runqing@staff.weibo.com",
17
+ "root": "/data4/runqing/dLLM-RL",
18
+ "host": "b1301b38c75f",
19
+ "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H800",
23
+ "gpu_count": 8,
24
+ "disk": {
25
+ "/": {
26
+ "total": "806578724864",
27
+ "used": "572337045504"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "2164113813504"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H800",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
40
+ },
41
+ {
42
+ "name": "NVIDIA H800",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
47
+ },
48
+ {
49
+ "name": "NVIDIA H800",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
54
+ },
55
+ {
56
+ "name": "NVIDIA H800",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
61
+ },
62
+ {
63
+ "name": "NVIDIA H800",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
68
+ },
69
+ {
70
+ "name": "NVIDIA H800",
71
+ "memoryTotal": "85520809984",
72
+ "cudaCores": 16896,
73
+ "architecture": "Hopper",
74
+ "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
75
+ },
76
+ {
77
+ "name": "NVIDIA H800",
78
+ "memoryTotal": "85520809984",
79
+ "cudaCores": 16896,
80
+ "architecture": "Hopper",
81
+ "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
82
+ },
83
+ {
84
+ "name": "NVIDIA H800",
85
+ "memoryTotal": "85520809984",
86
+ "cudaCores": 16896,
87
+ "architecture": "Hopper",
88
+ "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
89
+ }
90
+ ],
91
+ "cudaVersion": "12.4",
92
+ "writerId": "sr0a7mkap2eiuw8cpbm7o5idgj2l6pki"
93
+ }
wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_runtime":1036,"_wandb":{"runtime":1036}}
wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-24T16:44:25.780084766+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_w7chab5/port-1691523.txt","pid":1691523,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-03-24T16:44:25.780884873+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1691523}
3
+ {"time":"2026-03-24T16:44:25.780852026+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1691523-1692232-1524229333/socket","Net":"unix"}}
4
+ {"time":"2026-03-24T16:44:25.961716708+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-03-24T16:44:25.970032581+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"eb44q2tb","id":"1(@)"}
6
+ {"time":"2026-03-24T16:44:30.275637457+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"eb44q2tb","id":"1(@)"}
7
+ {"time":"2026-03-24T16:44:35.492970552+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"pfjk4x7e02ih"}
8
+ {"time":"2026-03-24T17:01:46.448086485+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
9
+ {"time":"2026-03-24T17:01:46.448222721+08:00","level":"INFO","msg":"server is shutting down"}
10
+ {"time":"2026-03-24T17:01:46.448192503+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
11
+ {"time":"2026-03-24T17:01:46.448403899+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
12
+ {"time":"2026-03-24T17:01:46.448442769+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1691523-1692232-1524229333/socket","Net":"unix"}}
13
+ {"time":"2026-03-24T17:01:46.588524788+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
14
+ {"time":"2026-03-24T17:01:46.588559276+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
15
+ {"time":"2026-03-24T17:01:46.588574496+08:00","level":"INFO","msg":"server is closed"}
wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-03-24T16:44:25.970209487+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
2
+ {"time":"2026-03-24T16:44:30.272794359+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
3
+ {"time":"2026-03-24T16:44:30.274781439+08:00","level":"INFO","msg":"stream: created new stream","id":"eb44q2tb"}
4
+ {"time":"2026-03-24T16:44:30.274957975+08:00","level":"INFO","msg":"handler: started","stream_id":"eb44q2tb"}
5
+ {"time":"2026-03-24T16:44:30.275605838+08:00","level":"INFO","msg":"writer: started","stream_id":"eb44q2tb"}
6
+ {"time":"2026-03-24T16:44:30.275607151+08:00","level":"INFO","msg":"stream: started","id":"eb44q2tb"}
7
+ {"time":"2026-03-24T16:44:30.275615082+08:00","level":"INFO","msg":"sender: started","stream_id":"eb44q2tb"}
8
+ {"time":"2026-03-24T16:44:30.27762458+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
9
+ {"time":"2026-03-24T16:44:35.521989789+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b3af0f9bd2bc9a1d25e7c95f703d6e7f082c92a5ddeaa4bb217a9b7cfe80832"}
10
+ {"time":"2026-03-24T16:44:35.522055528+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=aaee6c8449b2ae1a780ad0c20f3b6210d18b6720e515f80b6266904d04c19233"}
11
+ {"time":"2026-03-24T16:44:35.522099395+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch</Resource><RequestId>189FBA88EFF948C5</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b3af0f9bd2bc9a1d25e7c95f703d6e7f082c92a5ddeaa4bb217a9b7cfe80832, Size: 12686}"}
12
+ {"time":"2026-03-24T16:44:35.522145559+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json</Resource><RequestId>189FBA88F07C428E</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=aaee6c8449b2ae1a780ad0c20f3b6210d18b6720e515f80b6266904d04c19233, Size: 2636}"}
13
+ {"time":"2026-03-24T16:44:35.524085548+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=638a1b72719105e2fa963dfad98344f6b9593e0368c367c1724abafa9f17d52a"}
14
+ {"time":"2026-03-24T16:44:35.52450294+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d633c46405ba4ce40f9454aeca31c83c8d300c6eae3823e88f75b255c88ec8a"}
15
+ {"time":"2026-03-24T16:44:35.525507802+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch</Resource><RequestId>189FBA88F08CB76A</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=638a1b72719105e2fa963dfad98344f6b9593e0368c367c1724abafa9f17d52a, Size: 12686}"}
16
+ {"time":"2026-03-24T16:44:35.52555316+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt</Resource><RequestId>189FBA88F01C4C5D</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d633c46405ba4ce40f9454aeca31c83c8d300c6eae3823e88f75b255c88ec8a, Size: 2966}"}
17
+ {"time":"2026-03-24T16:44:35.533015823+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4421e08beab520f2a67729be80eff77a1e13d1128016b44b9b31a9397524de55"}
18
+ {"time":"2026-03-24T16:44:35.533064379+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py</Resource><RequestId>189FBA88F09E62CB</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4421e08beab520f2a67729be80eff77a1e13d1128016b44b9b31a9397524de55, Size: 32189}"}
19
+ {"time":"2026-03-24T17:01:46.448220293+08:00","level":"INFO","msg":"stream: closing","id":"eb44q2tb"}
20
+ {"time":"2026-03-24T17:01:46.543269717+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e3cd94ab34d1b5cb38d2d62928042acf4a195aacb875e5b3ce61af7e133f5d6c"}
21
+ {"time":"2026-03-24T17:01:46.543380827+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log</Resource><RequestId>189FBB78FE1F23F7</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e3cd94ab34d1b5cb38d2d62928042acf4a195aacb875e5b3ce61af7e133f5d6c, Size: 2682}"}
22
+ {"time":"2026-03-24T17:01:46.544795606+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=37c1c8b11290535a5dccb65faaf1b5e16cbc96ec65898ab418b2ed23847cabd0"}
23
+ {"time":"2026-03-24T17:01:46.544900322+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json</Resource><RequestId>189FBB78FD79ECBE</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=37c1c8b11290535a5dccb65faaf1b5e16cbc96ec65898ab418b2ed23847cabd0, Size: 43}"}
24
+ {"time":"2026-03-24T17:01:46.546082064+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a171387400e366cd3f49b5662ca5671bbf8af4bf04a935ca313d9b14507b6cf1"}
25
+ {"time":"2026-03-24T17:01:46.546183734+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml</Resource><RequestId>189FBB78FDB1FAB3</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a171387400e366cd3f49b5662ca5671bbf8af4bf04a935ca313d9b14507b6cf1, Size: 7766}"}
26
+ {"time":"2026-03-24T17:01:46.546930214+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
27
+ {"time":"2026-03-24T17:01:46.587769696+08:00","level":"INFO","msg":"handler: closed","stream_id":"eb44q2tb"}
28
+ {"time":"2026-03-24T17:01:46.587910039+08:00","level":"INFO","msg":"sender: closed","stream_id":"eb44q2tb"}
29
+ {"time":"2026-03-24T17:01:46.587937144+08:00","level":"INFO","msg":"stream: closed","id":"eb44q2tb"}
wandb/run-20260324_164425-eb44q2tb/logs/debug.log ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
2
+ 2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Configure stats pid to 1691523
3
+ 2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/logs/debug.log
5
+ 2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log
6
+ 2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():844] calling init triggers
7
+ 2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
+ config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}}
9
+ 2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():892] starting backend
10
+ 2026-03-24 16:44:25,961 INFO MainThread:1691523 [wandb_init.py:init():895] sending inform_init request
11
+ 2026-03-24 16:44:25,966 INFO MainThread:1691523 [wandb_init.py:init():903] backend started and connected
12
+ 2026-03-24 16:44:25,967 INFO MainThread:1691523 [wandb_init.py:init():973] updated telemetry
13
+ 2026-03-24 16:44:25,973 INFO MainThread:1691523 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
+ 2026-03-24 16:44:30,384 INFO MainThread:1691523 [wandb_init.py:init():1042] starting run threads in backend
15
+ 2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_console_start():2524] atexit reg
16
+ 2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-03-24 16:44:30,491 INFO MainThread:1691523 [wandb_init.py:init():1082] run started, returning control to user process
20
+ 2026-03-24 16:44:30,492 INFO MainThread:1691523 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'eb44q2tb', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 20, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 15, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 64, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'}
21
+ 2026-03-24 17:01:46,448 INFO wandb-AsyncioManager-main:1691523 [service_client.py:_forward_responses():134] Reached EOF.
22
+ 2026-03-24 17:01:46,448 INFO wandb-AsyncioManager-main:1691523 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py ADDED
@@ -0,0 +1,825 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
4
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
5
+ from datetime import timedelta
6
+ import json
7
+ import logging
8
+ import math
9
+ import shutil
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Dict, List, Any
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from torch.optim import AdamW
18
+ from torch.utils.data import Dataset, DataLoader
19
+
20
+ from omegaconf import OmegaConf
21
+ import wandb
22
+
23
+ from accelerate import Accelerator
24
+ from accelerate.logging import get_logger
25
+ from accelerate.utils import set_seed
26
+
27
+ from train.utils import get_config, flatten_omega_conf, AverageMeter
28
+ from models.lr_schedulers import get_scheduler
29
+ from models.logging import set_verbosity_info, set_verbosity_error
30
+
31
+ from transformers.utils import logging as hf_logging
32
+ hf_logging.set_verbosity_error()
33
+
34
+ from llava.model.builder import load_pretrained_model
35
+ from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord
36
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
37
+ from llava.conversation import conv_templates
38
+
39
+ from termcolor import cprint
40
+
41
+ logger = get_logger(__name__, log_level="INFO")
42
+
43
+
44
+ def collapse_k_unique(lst, k: int):
45
+ if k <= 0:
46
+ raise ValueError("k must be > 0")
47
+ uniq = sorted(set(lst))
48
+ mapping = {}
49
+ n = len(uniq)
50
+ for idx, val in enumerate(uniq):
51
+ group = idx // k
52
+ end_idx = min((group + 1) * k - 1, n - 1)
53
+ rep = uniq[end_idx]
54
+ mapping[val] = rep
55
+ return [mapping[x] for x in lst]
56
+
57
+
58
+ def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str:
59
+ import copy
60
+ conv = copy.deepcopy(conv_templates["llava_llada"])
61
+ conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text)
62
+ conv.append_message(conv.roles[1], response_text)
63
+ return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
64
+
65
+
66
+ def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor:
67
+ conv_prompt = build_llava_conv_prompt(prompt_text, response_text)
68
+ ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
69
+ return ids.squeeze(0)
70
+
71
+
72
+ def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16):
73
+ from PIL import Image
74
+ from types import SimpleNamespace
75
+
76
+ if isinstance(model_cfg, dict):
77
+ model_cfg = SimpleNamespace(**model_cfg)
78
+
79
+ if media_type == "image":
80
+ image = Image.open(abs_path).convert("RGB")
81
+ if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None:
82
+ setattr(model_cfg, "image_aspect_ratio", "pad")
83
+ if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None:
84
+ setattr(model_cfg, "mm_patch_merge_type", "flat")
85
+
86
+ image_tensor_list = process_images([image], image_processor, model_cfg)
87
+ image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list]
88
+ image_sizes = [image.size]
89
+ return image_tensor_list, image_sizes, None
90
+
91
+ elif media_type == "video":
92
+ if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1)
93
+ if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16)
94
+ if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True)
95
+
96
+ video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg)
97
+ image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"]
98
+ image_tensor = image_tensor.to(dtype=dtype, device=device)
99
+
100
+ if isinstance(video_frames, np.ndarray) and len(video_frames) > 0:
101
+ v_size = (video_frames.shape[2], video_frames.shape[1])
102
+ else:
103
+ v_size = (0, 0)
104
+
105
+ return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample)
106
+
107
+
108
+ def left_pad_1d(x: torch.Tensor, L: int, pad_val: int):
109
+ out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device)
110
+ out[-x.numel():] = x
111
+ return out
112
+
113
+
114
+ def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0):
115
+ if x.size(0) == L:
116
+ return x
117
+ out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device)
118
+ out[-x.size(0):] = x
119
+ return out
120
+
121
+
122
+ def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
123
+ p = sample.get(f"{media_type}_abs_path")
124
+ if p is None:
125
+ p = sample.get(media_type) or sample.get(f"{media_type}_path")
126
+ if p is None:
127
+ return None
128
+ if not os.path.isabs(p) and dataset_root:
129
+ p = os.path.join(dataset_root, p)
130
+ try:
131
+ return os.path.abspath(p)
132
+ except Exception:
133
+ return p
134
+
135
+
136
+ class RLVariantDatasetLLADAV(Dataset):
137
+ def __init__(self, samples: List[dict], config):
138
+ super().__init__()
139
+ self.samples = samples
140
+ self.config = config
141
+ self.variants: List[dict] = []
142
+
143
+ # logp_old_tok mapping: idx -> tensor
144
+ self.logp_old_tok: Dict[int, torch.Tensor] = {}
145
+
146
+ self._build_variants()
147
+
148
+ def _build_variants(self):
149
+ sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
150
+ question_prompt = "\nOnly give the best option."
151
+ multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags."
152
+
153
+ rng = np.random.default_rng(seed=self.config.training.seed or 42)
154
+ idx_counter = 0
155
+
156
+ for sid, s in enumerate(self.samples):
157
+ # Rollout produces "extracted_output" or "response", fall back if needed
158
+ response_text = s.get("response", s.get("extracted_output", ""))
159
+ if isinstance(response_text, list) and len(response_text) > 0:
160
+ response_text = response_text[0] # Just in case it's nested
161
+
162
+ # If options exist (video multiple choice), construct full prompt
163
+ if "options" in s:
164
+ prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt
165
+ else:
166
+ prompt_text = s["prompt"]
167
+
168
+ step_map = s.get("step_map", [])
169
+
170
+ if self.config.training.method == "TraceRL":
171
+ order_list = collapse_k_unique(step_map, self.config.training.shrink)
172
+ uniq_steps = sorted(set(order_list))
173
+ for st in uniq_steps:
174
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None))
175
+ idx_counter += 1
176
+
177
+ elif self.config.training.method == "random_masking":
178
+ for _ in range(self.config.training.mask_times_per_sample):
179
+ t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
180
+ mask_seed = int(rng.integers(0, 2**31 - 1))
181
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed))
182
+ idx_counter += 1
183
+
184
+ elif self.config.training.method == "coupled":
185
+ for _ in range(self.config.training.mask_times_per_sample):
186
+ t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p))
187
+ mask_seed = int(rng.integers(0, 2**31 - 1))
188
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed))
189
+ idx_counter += 1
190
+ self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed))
191
+ idx_counter += 1
192
+ else:
193
+ raise ValueError(f"Unknown RL method: {self.config.training.method}")
194
+
195
+ def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed):
196
+ return dict(
197
+ idx=idx,
198
+ sample_id=sid,
199
+ prompt=prompt,
200
+ response=response,
201
+ image_abs_path=s.get("image_abs_path", None),
202
+ video_abs_path=s.get("video_abs_path", None),
203
+ reward=s.get("reward", 0.0),
204
+ var_type=var_type,
205
+ var_param=var_param,
206
+ step_map=step_map,
207
+ mask_seed=mask_seed,
208
+ )
209
+
210
+ def __len__(self):
211
+ return len(self.variants)
212
+
213
+ def __getitem__(self, idx):
214
+ return self.variants[idx]
215
+
216
+
217
+ def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]:
218
+ out: Dict[str, List[Any]] = {}
219
+ keys = batch[0].keys()
220
+ for k in keys:
221
+ out[k] = [b[k] for b in batch]
222
+ return out
223
+
224
+
225
+ @torch.no_grad()
226
+ def prepare_batch_noisy_embeds_rl(
227
+ accelerator,
228
+ model,
229
+ tokenizer,
230
+ image_processor,
231
+ batch: Dict[str, List[Any]],
232
+ mask_id: int,
233
+ dataset_media_root: str | None,
234
+ ) -> tuple | None:
235
+
236
+ device = accelerator.device
237
+ bsz = len(batch["idx"])
238
+
239
+ group: Dict[int, List[int]] = {}
240
+ for i, sid in enumerate(batch["sample_id"]):
241
+ group.setdefault(int(sid), []).append(i)
242
+
243
+ sample_cache: Dict[int, Dict[str, Any]] = {}
244
+ for sid, pos_list in group.items():
245
+ first_pos = pos_list[0]
246
+ prompt_text = batch["prompt"][first_pos]
247
+ response_text = batch["response"][first_pos]
248
+
249
+ sample_stub = {
250
+ "image_abs_path": batch.get("image_abs_path", [None])[first_pos],
251
+ "video_abs_path": batch.get("video_abs_path", [None])[first_pos],
252
+ }
253
+
254
+ media_type = None
255
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video")
256
+ if p_abs is not None:
257
+ media_type = "video"
258
+ else:
259
+ p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image")
260
+ if p_abs is not None:
261
+ media_type = "image"
262
+
263
+ img_list, image_sizes, modalities = None, None, []
264
+ if p_abs is not None:
265
+ img_list, image_sizes, video_stats = load_and_process_media(
266
+ p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype
267
+ )
268
+ modalities = [media_type]
269
+
270
+ if media_type == "video" and getattr(model.config, "add_time_instruction", False):
271
+ video_time, frame_time, num_frames = video_stats
272
+ time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video."
273
+ clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip()
274
+ prompt_text = f"{time_instruction}\n{clean_prompt}"
275
+
276
+ input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device)
277
+ end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False)
278
+ seq_len = len(end_header_seq)
279
+ pos_eh = -1
280
+ if seq_len > 0 and input_ids_text.numel() >= seq_len:
281
+ ids_list = input_ids_text.tolist()
282
+ for i in range(len(ids_list) - seq_len, -1, -1):
283
+ if ids_list[i:i + seq_len] == end_header_seq:
284
+ pos_eh = i + seq_len - 1
285
+ break
286
+ assert pos_eh != -1, " <|end_header_id|> not found"
287
+ start_pos = pos_eh + 1
288
+
289
+ labels_text = input_ids_text.clone()
290
+ labels_text[:start_pos] = -100
291
+
292
+ (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal(
293
+ input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes
294
+ )
295
+ clean_embeds = clean_embeds.squeeze(0)
296
+ new_labels = new_labels.squeeze(0)
297
+
298
+ eot_id = 126348
299
+ eot_indices = (new_labels == eot_id).nonzero(as_tuple=False)
300
+ if eot_indices.numel() > 0:
301
+ last_eot_pos = eot_indices[-1, 0]
302
+ new_labels[last_eot_pos:] = -100
303
+ asst_mask = (new_labels != -100)
304
+ asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1)
305
+
306
+ sample_cache[sid] = dict(
307
+ clean_embeds=clean_embeds,
308
+ new_labels=new_labels,
309
+ asst_idx=asst_idx,
310
+ )
311
+
312
+ lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)]
313
+ Lmax = max(lengths)
314
+ mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0)
315
+
316
+ noisy_list, labels_list, p_mask_list, attn_list = [], [], [], []
317
+ reward_list, valid_idxs = [], []
318
+
319
+ for i in range(bsz):
320
+ sid = int(batch["sample_id"][i])
321
+ var_type = batch["var_type"][i]
322
+ var_param = batch["var_param"][i]
323
+
324
+ clean_embeds = sample_cache[sid]["clean_embeds"].clone()
325
+ new_labels = sample_cache[sid]["new_labels"].clone()
326
+ asst_idx = sample_cache[sid]["asst_idx"]
327
+
328
+ L = clean_embeds.size(0)
329
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
330
+ M = asst_idx.numel()
331
+
332
+ if M == 0:
333
+ continue # skip badly formatted inputs
334
+
335
+ if var_type == "TraceRL":
336
+ step_map = batch["step_map"][i]
337
+ order = torch.as_tensor(step_map, device=device)
338
+ m_len = min(M, order.numel())
339
+ order_full = torch.full((L,), -1, dtype=torch.long, device=device)
340
+ order_full[asst_idx[:m_len]] = order[:m_len]
341
+ step_val = int(var_param)
342
+
343
+ tgt_mask = (order_full == step_val)
344
+ pmask = tgt_mask
345
+ mask_pos = (order_full >= step_val)
346
+
347
+ noisy_embeds = clean_embeds.clone()
348
+ noisy_embeds[mask_pos] = mask_embed
349
+
350
+ elif var_type == "random_masking" or var_type.startswith("coupled"):
351
+ if M == 0:
352
+ # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward
353
+ noisy_embeds = clean_embeds.clone()
354
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
355
+ else:
356
+ t = float(var_param)
357
+ # 更安全的 seed 获取方式
358
+ mask_seed = batch.get("mask_seed", [None] * bsz)[i]
359
+ if mask_seed is None:
360
+ mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff)
361
+
362
+ gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
363
+ rnd_cpu = torch.rand(M, generator=gen) < t
364
+
365
+ # 如果是 coupled_comp (对比样本),需要取反
366
+ if var_type == "coupled_comp":
367
+ rnd_cpu = ~rnd_cpu
368
+
369
+ rnd = rnd_cpu.to(device)
370
+
371
+ # 🌟 上帝之手:强行兜底逻辑
372
+ # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁
373
+ if not rnd.any():
374
+ # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反,
375
+ # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。
376
+ # 这里统一下一个兜底,非常安全。
377
+ rand_pos = torch.randint(0, M, (1,), device=device)
378
+ rnd[rand_pos] = True
379
+
380
+ pmask = torch.zeros(L, dtype=torch.bool, device=device)
381
+ pmask[asst_idx] = rnd
382
+
383
+ noisy_embeds = clean_embeds.clone()
384
+ noisy_embeds[pmask] = mask_embed
385
+
386
+ # elif var_type == "random_masking" or var_type.startswith("coupled"):
387
+ # t = float(var_param)
388
+ # mask_seed = batch["mask_seed"][i]
389
+ # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed))
390
+ # rnd_cpu = torch.rand(M, generator=gen) < t
391
+
392
+ # if var_type == "coupled_comp":
393
+ # rnd_cpu = ~rnd_cpu
394
+
395
+ # pmask = torch.zeros(L, dtype=torch.bool, device=device)
396
+ # pmask[asst_idx] = rnd_cpu.to(device)
397
+
398
+ # noisy_embeds = clean_embeds.clone()
399
+ # noisy_embeds[pmask] = mask_embed
400
+
401
+ if not pmask.any():
402
+ print('************not pmask.any()**********')
403
+ continue
404
+
405
+ noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0))
406
+ labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100))
407
+ p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool))
408
+ attn = torch.zeros(Lmax, dtype=torch.bool, device=device)
409
+ attn[-L:] = True
410
+ attn_list.append(attn)
411
+ reward_list.append(float(batch["reward"][i]))
412
+ valid_idxs.append(batch["idx"][i])
413
+
414
+ if len(noisy_list) == 0:
415
+ return None
416
+
417
+ noisy_batch = torch.stack(noisy_list, dim=0)
418
+ labels_batch = torch.stack(labels_list, dim=0)
419
+ p_mask_batch = torch.stack(p_mask_list, dim=0)
420
+ attn_batch = torch.stack(attn_list, dim=0)
421
+ reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32)
422
+
423
+ return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs
424
+
425
+
426
+ @torch.no_grad()
427
+ def compute_logp_old_tok_parallel(
428
+ accelerator,
429
+ model,
430
+ tokenizer,
431
+ image_processor,
432
+ dataset,
433
+ dl,
434
+ mask_id,
435
+ config):
436
+
437
+ model.eval()
438
+ logger.info("Computing reference log probabilities (old_lp) for all variants...")
439
+
440
+ for batch in dl:
441
+ out = prepare_batch_noisy_embeds_rl(
442
+ accelerator=accelerator,
443
+ model=model,
444
+ tokenizer=tokenizer,
445
+ image_processor=image_processor,
446
+ batch=batch,
447
+ mask_id=mask_id,
448
+ dataset_media_root=config.dataset.get("image_root", None),
449
+ )
450
+ if out is None:
451
+ continue
452
+
453
+ noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out
454
+
455
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True)
456
+ logits = model.lm_head(outputs.last_hidden_state).float()
457
+
458
+ log_probs = F.log_softmax(logits, dim=-1)
459
+ safe_labels = labels_batch.clone()
460
+ safe_labels[labels_batch == -100] = 0
461
+ tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
462
+
463
+ for i, idx in enumerate(valid_idxs):
464
+ dataset.logp_old_tok[idx] = tok_lp[i].float().cpu()
465
+
466
+ accelerator.wait_for_everyone()
467
+ model.train()
468
+
469
+
470
+ def save_checkpoint(model, tokenizer, config, accelerator, name):
471
+ output_dir = Path(config.experiment.project)
472
+ output_dir.mkdir(parents=True, exist_ok=True)
473
+
474
+ checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None)
475
+
476
+ if accelerator.is_main_process and checkpoints_total_limit is not None:
477
+ ckpts = sorted(
478
+ [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")],
479
+ key=lambda p: int(p.name.split("-")[1]),
480
+ )
481
+ if len(ckpts) >= checkpoints_total_limit:
482
+ to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1]
483
+ logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}")
484
+ for p in to_remove:
485
+ shutil.rmtree(p, ignore_errors=True)
486
+
487
+ save_base = output_dir / "ckpt"
488
+ save_base.mkdir(exist_ok=True)
489
+
490
+ model_to_save = accelerator.unwrap_model(model)
491
+ state_dict = accelerator.get_state_dict(model)
492
+
493
+ if accelerator.is_main_process:
494
+ model_to_save.save_pretrained(
495
+ save_base / name,
496
+ save_function=accelerator.save,
497
+ state_dict=state_dict,
498
+ safe_serialization=True,
499
+ )
500
+ tokenizer.save_pretrained(str(save_base / name))
501
+
502
+ metadata = {
503
+ "save_time": time.strftime("%Y-%m-%d %H:%M:%S"),
504
+ }
505
+ with (save_base / "metadata.json").open("w") as f:
506
+ json.dump(metadata, f, indent=2)
507
+
508
+ logger.info(f"Saved model + tokenizer to {save_base / name}")
509
+
510
+
511
+ def main():
512
+ config = get_config()
513
+
514
+ project_name = config.experiment.project
515
+ if config.experiment.current_epoch == 1:
516
+ pretrained_model = config.model.pretrained_model
517
+ else:
518
+ pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name
519
+
520
+ if config.training.enable_tf32:
521
+ torch.backends.cuda.matmul.allow_tf32 = True
522
+ torch.backends.cudnn.benchmark = True
523
+ torch.backends.cudnn.deterministic = False
524
+
525
+ config.experiment.logging_dir = str(Path(config.experiment.project) / "logs")
526
+ accelerator = Accelerator(
527
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
528
+ mixed_precision=config.training.mixed_precision,
529
+ log_with='wandb',
530
+ project_dir=config.experiment.logging_dir,
531
+ split_batches=True,
532
+ )
533
+
534
+ logging.basicConfig(
535
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
536
+ datefmt="%m/%d/%Y %H:%M:%S",
537
+ level=logging.INFO,
538
+ )
539
+ logger.info(accelerator.state, main_process_only=False)
540
+ if accelerator.is_local_main_process:
541
+ set_verbosity_info()
542
+ else:
543
+ set_verbosity_error()
544
+
545
+ if accelerator.is_main_process:
546
+ resume_wandb_run = config.wandb.resume
547
+ run_id = config.wandb.get("run_id", None)
548
+ if run_id is None:
549
+ resume_wandb_run = False
550
+ run_id = wandb.util.generate_id()
551
+ config.wandb.run_id = run_id
552
+
553
+ wandb_init_kwargs = dict(
554
+ name=config.experiment.project,
555
+ id=run_id,
556
+ resume=resume_wandb_run,
557
+ entity=config.wandb.get("entity", None),
558
+ config_exclude_keys=[],
559
+ )
560
+ wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
561
+ wandb_config.pop("experiment.resume_from_checkpoint", None)
562
+
563
+ accelerator.init_trackers(
564
+ config.experiment.project,
565
+ config=wandb_config,
566
+ init_kwargs={"wandb": wandb_init_kwargs},
567
+ )
568
+
569
+ if accelerator.is_main_process:
570
+ os.makedirs(config.experiment.project, exist_ok=True)
571
+ config_path = Path(config.experiment.project) / "config.yaml"
572
+ logging.info(f"Saving config to {config_path}")
573
+ OmegaConf.save(config, config_path)
574
+
575
+ if config.training.seed is not None:
576
+ set_seed(config.training.seed)
577
+
578
+ logger.info("Loading LLaDA-V model and tokenizer")
579
+
580
+ tokenizer, model, image_processor, _ = load_pretrained_model(
581
+ pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device
582
+ )
583
+ model = model.to(accelerator.device)
584
+
585
+ special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]}
586
+ num_new = tokenizer.add_special_tokens(special_tokens)
587
+ if num_new > 0:
588
+ model.resize_token_embeddings(len(tokenizer))
589
+ if tokenizer.pad_token is None:
590
+ tokenizer.pad_token = tokenizer.eos_token
591
+
592
+ if getattr(config.training, "gradient_checkpointing_enable", False):
593
+ try:
594
+ model.enable_input_require_grads()
595
+ except Exception:
596
+ pass
597
+ try:
598
+ model.gradient_checkpointing_enable()
599
+ except Exception:
600
+ pass
601
+
602
+ mask_id = 126336
603
+
604
+ optimizer_config = config.optimizer.params
605
+ no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"]
606
+ optimizer_grouped_parameters = [
607
+ {
608
+ "params": [p for n, p in model.named_parameters() if
609
+ p.requires_grad and not any(nd in n for nd in no_decay)],
610
+ "weight_decay": optimizer_config.weight_decay,
611
+ },
612
+ {
613
+ "params": [p for n, p in model.named_parameters() if
614
+ p.requires_grad and any(nd in n for nd in no_decay)],
615
+ "weight_decay": 0.0,
616
+ },
617
+ ]
618
+
619
+ optimizer_type = config.optimizer.name
620
+ if optimizer_type == "adamw":
621
+ optimizer = AdamW(
622
+ optimizer_grouped_parameters,
623
+ lr=optimizer_config.learning_rate,
624
+ betas=(optimizer_config.beta1, optimizer_config.beta2),
625
+ weight_decay=optimizer_config.weight_decay,
626
+ eps=optimizer_config.epsilon,
627
+ )
628
+ else:
629
+ raise ValueError(f"Optimizer {optimizer_type} not supported")
630
+
631
+
632
+ logger.info("Preprocessing Data")
633
+
634
+ with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f:
635
+ dataset_load = json.load(f)
636
+
637
+ dataset_lm = RLVariantDatasetLLADAV(dataset_load, config)
638
+
639
+ logger.info("Preparing model, optimizer and dataloaders")
640
+
641
+ total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps
642
+ num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm)
643
+ num_train_epochs = config.training.num_train_epochs
644
+ max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1
645
+
646
+ lr_scheduler = get_scheduler(
647
+ config.lr_scheduler.scheduler,
648
+ optimizer=optimizer,
649
+ num_training_steps=max_train_steps,
650
+ num_warmup_steps=config.lr_scheduler.params.warmup_steps,
651
+ min_lr_scale=config.lr_scheduler.params.min_lr_scale
652
+ )
653
+
654
+ train_dataloader_lm = DataLoader(
655
+ dataset_lm,
656
+ batch_size=config.training.batch_size_lm,
657
+ sampler=None,
658
+ collate_fn=collate_variants,
659
+ num_workers=0,
660
+ shuffle=False
661
+ )
662
+
663
+ model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare(
664
+ model, optimizer, lr_scheduler, train_dataloader_lm
665
+ )
666
+
667
+ logger.info("***** Running inference *****")
668
+ compute_logp_old_tok_parallel(
669
+ accelerator,
670
+ model,
671
+ tokenizer,
672
+ image_processor,
673
+ dataset_lm,
674
+ train_dataloader_lm,
675
+ mask_id=mask_id,
676
+ config=config
677
+ )
678
+
679
+ logger.info("***** Running training *****")
680
+ logger.info(f" Num variants = {len(dataset_lm)}")
681
+ logger.info(f" Num training steps = {max_train_steps}")
682
+ logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}")
683
+ logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}")
684
+ logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}")
685
+
686
+ first_epoch = 0
687
+ data_time_m = AverageMeter()
688
+ end = time.time()
689
+ grad_accum = config.training.gradient_accumulation_steps
690
+
691
+ def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok):
692
+ outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True)
693
+ logits = model.lm_head(outputs.last_hidden_state).float()
694
+
695
+ B, T, V = logits.shape
696
+ adv = torch.as_tensor(adv, device=noisy_embeds.device).detach()
697
+
698
+ log_probs = F.log_softmax(logits, dim=-1)
699
+ safe_labels = labels.clone()
700
+ safe_labels[labels == -100] = 0
701
+ logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1)
702
+
703
+ ratio = torch.exp(logp_new_tok - logp_old_tok)
704
+ clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps)
705
+ adv_tok = adv.unsqueeze(1)
706
+
707
+ surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok)
708
+ surrogate_tok = surrogate_tok * p_mask_lm
709
+ num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1)
710
+ surrogate_tok = surrogate_tok.sum(dim=1) / num_mask
711
+ policy_loss = - (surrogate_tok.sum() / B)
712
+
713
+ # KL penalty
714
+ kl_loss = torch.tensor(0.0, device=policy_loss.device)
715
+ if config.training.beta > 0:
716
+ kl_seq = logp_new_tok - logp_old_tok
717
+ if config.training.use_kl_estimator_k3:
718
+ kl_seq = (-kl_seq).exp() - 1.0 + kl_seq
719
+ kl_seq = (kl_seq * p_mask_lm).sum(dim=1)
720
+ kl_loss = config.training.beta * kl_seq.sum() / B
721
+ total_loss = policy_loss + kl_loss
722
+ else:
723
+ total_loss = policy_loss
724
+
725
+ return total_loss
726
+
727
+ from tqdm.auto import tqdm
728
+
729
+ cprint("start rl training...", "red")
730
+ for epoch in range(first_epoch, num_train_epochs):
731
+ model.train()
732
+ progress_bar = tqdm(
733
+ train_dataloader_lm,
734
+ desc=f"Epoch {epoch+1}/{num_train_epochs}",
735
+ disable=not accelerator.is_local_main_process,
736
+ dynamic_ncols=True,
737
+ leave=True
738
+ )
739
+
740
+ step = 0
741
+ for step, batch in enumerate(progress_bar, start=1):
742
+ data_time_m.update(time.time() - end)
743
+
744
+ out = prepare_batch_noisy_embeds_rl(
745
+ accelerator=accelerator,
746
+ model=model,
747
+ tokenizer=tokenizer,
748
+ image_processor=image_processor,
749
+ batch=batch,
750
+ mask_id=mask_id,
751
+ dataset_media_root=config.dataset.get("image_root", None),
752
+ )
753
+
754
+ if out is None:
755
+ continue
756
+
757
+ noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out
758
+
759
+ # Align old log probabilities shape with current noisy_embeds
760
+ L_current = noisy_embeds.shape[1]
761
+ old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device)
762
+ for i, idx in enumerate(valid_idxs):
763
+ old_t = dataset_lm.logp_old_tok[idx]
764
+ l_min = min(L_current, old_t.shape[0])
765
+ old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device)
766
+
767
+ loss_lm = forward_process(
768
+ noisy_embeds=noisy_embeds,
769
+ labels=labels,
770
+ p_mask_lm=p_mask_lm,
771
+ attn_mask=attn_mask,
772
+ adv=rewards,
773
+ logp_old_tok=old_lp_batch
774
+ )
775
+
776
+ loss_lm = loss_lm / grad_accum
777
+ accelerator.backward(loss_lm)
778
+
779
+ if step % grad_accum == 0:
780
+ current_lr = optimizer.param_groups[0]['lr']
781
+ # current_global_step = global_step_offset + (step // grad_accum)
782
+
783
+ accelerator.log({
784
+ "train/loss": loss_lm.item() * grad_accum,
785
+ "train/learning_rate": current_lr,
786
+ # "train/iteration": config.experiment.current_epoch,
787
+ }, step=(step // grad_accum))
788
+
789
+
790
+ if config.training.max_grad_norm is not None:
791
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
792
+ optimizer.step()
793
+ lr_scheduler.step()
794
+ optimizer.zero_grad(set_to_none=True)
795
+ torch.cuda.empty_cache()
796
+
797
+ # Catch remaining gradients
798
+ if step % grad_accum != 0:
799
+ current_lr = optimizer.param_groups[0]['lr']
800
+ # current_global_step = global_step_offset + (step // grad_accum)
801
+
802
+ accelerator.log({
803
+ "train/loss": loss_lm.item() * grad_accum,
804
+ "train/learning_rate": current_lr,
805
+ # "train/iteration": config.experiment.current_epoch,
806
+ }, step=(step // grad_accum))
807
+
808
+ if config.training.max_grad_norm is not None:
809
+ accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm)
810
+ optimizer.step()
811
+ lr_scheduler.step()
812
+ optimizer.zero_grad(set_to_none=True)
813
+ torch.cuda.empty_cache()
814
+
815
+ accelerator.wait_for_everyone()
816
+
817
+ save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name)
818
+ if config.experiment.current_epoch % config.experiment.save_every == 0:
819
+ save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}")
820
+
821
+ accelerator.end_training()
822
+
823
+
824
+ if __name__ == "__main__":
825
+ main()
wandb/run-20260325_141344-o1lsxhwu/files/config.yaml ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.25.0
4
+ code_path: code/train/rl_lladav.py
5
+ e:
6
+ l1zvo1vvzkyq01b0wblads1a5x42yuiv:
7
+ args:
8
+ - config=configs/rl_lladav_video_reversal.yaml
9
+ - experiment.current_epoch=27
10
+ codePath: train/rl_lladav.py
11
+ codePathLocal: train/rl_lladav.py
12
+ cpu_count: 96
13
+ cpu_count_logical: 192
14
+ cudaVersion: "12.4"
15
+ disk:
16
+ /:
17
+ total: "806578724864"
18
+ used: "572884140032"
19
+ email: shixi_runqing@staff.weibo.com
20
+ executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10
21
+ git:
22
+ commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
23
+ remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
24
+ gpu: NVIDIA H800
25
+ gpu_count: 8
26
+ gpu_nvidia:
27
+ - architecture: Hopper
28
+ cudaCores: 16896
29
+ memoryTotal: "85520809984"
30
+ name: NVIDIA H800
31
+ uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
32
+ - architecture: Hopper
33
+ cudaCores: 16896
34
+ memoryTotal: "85520809984"
35
+ name: NVIDIA H800
36
+ uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
37
+ - architecture: Hopper
38
+ cudaCores: 16896
39
+ memoryTotal: "85520809984"
40
+ name: NVIDIA H800
41
+ uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
42
+ - architecture: Hopper
43
+ cudaCores: 16896
44
+ memoryTotal: "85520809984"
45
+ name: NVIDIA H800
46
+ uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
47
+ - architecture: Hopper
48
+ cudaCores: 16896
49
+ memoryTotal: "85520809984"
50
+ name: NVIDIA H800
51
+ uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
52
+ - architecture: Hopper
53
+ cudaCores: 16896
54
+ memoryTotal: "85520809984"
55
+ name: NVIDIA H800
56
+ uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
57
+ - architecture: Hopper
58
+ cudaCores: 16896
59
+ memoryTotal: "85520809984"
60
+ name: NVIDIA H800
61
+ uuid: GPU-2775489f-18c8-2328-8829-c46238575379
62
+ - architecture: Hopper
63
+ cudaCores: 16896
64
+ memoryTotal: "85520809984"
65
+ name: NVIDIA H800
66
+ uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
67
+ host: b1301b38c75f
68
+ memory:
69
+ total: "2164113813504"
70
+ os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
71
+ program: /data4/runqing/dLLM-RL/train/rl_lladav.py
72
+ python: CPython 3.10.19
73
+ root: /data4/runqing/dLLM-RL
74
+ startedAt: "2026-03-25T06:13:44.186076Z"
75
+ writerId: l1zvo1vvzkyq01b0wblads1a5x42yuiv
76
+ m: []
77
+ python_version: 3.10.19
78
+ t:
79
+ "1":
80
+ - 1
81
+ - 5
82
+ - 11
83
+ - 41
84
+ - 49
85
+ - 53
86
+ - 71
87
+ - 98
88
+ "2":
89
+ - 1
90
+ - 5
91
+ - 11
92
+ - 41
93
+ - 49
94
+ - 51
95
+ - 53
96
+ - 71
97
+ - 98
98
+ "3":
99
+ - 2
100
+ - 13
101
+ - 14
102
+ - 61
103
+ "4": 3.10.19
104
+ "5": 0.25.0
105
+ "6": 4.52.4
106
+ "12": 0.25.0
107
+ "13": linux-x86_64
108
+ config:
109
+ value: configs/rl_lladav_video_reversal.yaml
110
+ dataset.data_type:
111
+ value: video_reverse
112
+ dataset.optimization_data:
113
+ value: rl_data
114
+ dataset.train_dataset:
115
+ value: video_reversal_rl_train
116
+ evaluation.batch_size:
117
+ value: 1
118
+ evaluation.block_size:
119
+ value: 32
120
+ evaluation.data_type:
121
+ value: video_reverse
122
+ evaluation.dynamic_threshold:
123
+ value: 0.9
124
+ evaluation.eval_dataset:
125
+ value: video_reversal_rl_eval
126
+ evaluation.further_horizon:
127
+ value: 128
128
+ evaluation.if_eval:
129
+ value: true
130
+ evaluation.max_gen_length:
131
+ value: 128
132
+ evaluation.num_response_per_task:
133
+ value: 3
134
+ evaluation.remasking_strategy.0:
135
+ value: low_confidence_static
136
+ evaluation.remasking_strategy.1:
137
+ value: low_confidence_dynamic
138
+ evaluation.steps:
139
+ value: 128
140
+ evaluation.target:
141
+ value: confidence
142
+ evaluation.temperature:
143
+ value: 0.1
144
+ evaluation.use_cache:
145
+ value: true
146
+ execute.num_chunk:
147
+ value: 128
148
+ experiment.current_epoch:
149
+ value: 27
150
+ experiment.deepspeed_file:
151
+ value: 1_node_8_gpus_deepspeed_zero2_rl
152
+ experiment.eval_every:
153
+ value: 5
154
+ experiment.function:
155
+ value: train
156
+ experiment.logging_dir:
157
+ value: rl_lladav_video_reversal/logs
158
+ experiment.node_index:
159
+ value: 0
160
+ experiment.num_node:
161
+ value: 1
162
+ experiment.project:
163
+ value: rl_lladav_video_reversal
164
+ experiment.save_every:
165
+ value: 10
166
+ experiment.start_from_scratch:
167
+ value: true
168
+ experiment.total_step:
169
+ value: 30
170
+ lr_scheduler.params.learning_rate:
171
+ value: 1e-06
172
+ lr_scheduler.params.min_lr_scale:
173
+ value: 1
174
+ lr_scheduler.params.warmup_steps:
175
+ value: 0
176
+ lr_scheduler.scheduler:
177
+ value: cosine
178
+ model.image_resolution:
179
+ value: 512
180
+ model.model_base:
181
+ value: lladav
182
+ model.optimized_name:
183
+ value: optimized
184
+ model.pretrained_model:
185
+ value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized
186
+ model.vq_model_path:
187
+ value: /data4/runqing/hf/magvitv2
188
+ optimizer.name:
189
+ value: adamw
190
+ optimizer.params.beta1:
191
+ value: 0.9
192
+ optimizer.params.beta2:
193
+ value: 0.999
194
+ optimizer.params.epsilon:
195
+ value: 1e-08
196
+ optimizer.params.learning_rate:
197
+ value: 1e-06
198
+ optimizer.params.scale_lr:
199
+ value: false
200
+ optimizer.params.weight_decay:
201
+ value: 0
202
+ reward.answer_must_in_box:
203
+ value: true
204
+ reward.strict_len_check:
205
+ value: false
206
+ rollout.batch_size:
207
+ value: 1
208
+ rollout.block_size:
209
+ value: 32
210
+ rollout.dynamic_threshold:
211
+ value: 0.9
212
+ rollout.further_horizon:
213
+ value: 128
214
+ rollout.max_gen_length:
215
+ value: 128
216
+ rollout.num_response_per_task:
217
+ value: 8
218
+ rollout.num_task_per_step:
219
+ value: 128
220
+ rollout.remasking_strategy:
221
+ value: low_confidence_dynamic
222
+ rollout.steps:
223
+ value: 128
224
+ rollout.target:
225
+ value: confidence
226
+ rollout.temperature:
227
+ value: 0.8
228
+ rollout.use_cache:
229
+ value: true
230
+ system.HF_HOME:
231
+ value: null
232
+ system.HTTP_PROXY:
233
+ value: null
234
+ system.base_dir:
235
+ value: /data4/runqing/dLLM-RL
236
+ system.env_name:
237
+ value: /data8/runqing/miniconda/envs/dllm-rl
238
+ system.envs_dir:
239
+ value: /data8/runqing/miniconda/envs/dllm-rl
240
+ training.batch_size_lm:
241
+ value: 1
242
+ training.beta:
243
+ value: 0.01
244
+ training.enable_tf32:
245
+ value: true
246
+ training.eps:
247
+ value: 0.2
248
+ training.gradient_accumulation_steps:
249
+ value: 2
250
+ training.lower_p:
251
+ value: 0.35
252
+ training.mask_times_per_sample:
253
+ value: 2
254
+ training.max_gen_length:
255
+ value: 128
256
+ training.max_grad_norm:
257
+ value: 1
258
+ training.max_prompt_len:
259
+ value: 512
260
+ training.method:
261
+ value: random_masking
262
+ training.mixed_precision:
263
+ value: bf16
264
+ training.num_train_epochs:
265
+ value: 1
266
+ training.post_num:
267
+ value: 1
268
+ training.seed:
269
+ value: 10086
270
+ training.shrink:
271
+ value: 8
272
+ training.upper_p:
273
+ value: 0.9
274
+ training.use_kl_estimator_k3:
275
+ value: true
276
+ wandb.entity:
277
+ value: null
278
+ wandb.resume:
279
+ value: auto
280
+ wandb.run_id:
281
+ value: o1lsxhwu
wandb/run-20260325_141344-o1lsxhwu/files/diff.patch ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..9f94fa7 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,87 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [
158
+ + {
159
+ + "name": "stdout",
160
+ + "output_type": "stream",
161
+ + "text": [
162
+ + "200\n"
163
+ + ]
164
+ + },
165
+ + {
166
+ + "ename": "",
167
+ + "evalue": "",
168
+ + "output_type": "error",
169
+ + "traceback": [
170
+ + "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
171
+ + "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
172
+ + "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
173
+ + "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
174
+ + ]
175
+ + }
176
+ + ],
177
+ + "source": [
178
+ + "import json\n",
179
+ + "import random\n",
180
+ + "random.seed(2026)\n",
181
+ + "\n",
182
+ + "\n",
183
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
184
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
185
+ + "save_data_list = []\n",
186
+ + "save_num = 0\n",
187
+ + "with open(data_path, 'r') as fin, \\\n",
188
+ + " open(save_path, 'w') as fout:\n",
189
+ + " raw_data = json.load(fin)\n",
190
+ + "\n",
191
+ + " # random.shuffle(raw_data)\n",
192
+ + " for did, data in enumerate(raw_data):\n",
193
+ + " # if did > 1999:\n",
194
+ + " if save_num < 200:\n",
195
+ + " save_data_list.append(data)\n",
196
+ + " save_num += 1\n",
197
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
198
+ + " print(len(save_data_list))\n"
199
+ + ]
200
+ }
201
+ ],
202
+ "metadata": {
203
+ @@ -85,7 +166,7 @@
204
+ "name": "python",
205
+ "nbconvert_exporter": "python",
206
+ "pygments_lexer": "ipython3",
207
+ - "version": "3.10.18"
208
+ + "version": "3.10.19"
209
+ }
210
+ },
211
+ "nbformat": 4,
212
+ diff --git a/rl_v.py b/rl_v.py
213
+ index 8f2c0f1..79d2e5b 100644
214
+ --- a/rl_v.py
215
+ +++ b/rl_v.py
216
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
217
+ def env_prefix(env) -> str:
218
+ return (
219
+ "source ~/.bashrc && "
220
+ - f"source activate {env} && "
221
+ + "source /data8/runqing/miniconda/bin/activate && "
222
+ + "conda init && "
223
+ + f"conda activate {env} && "
224
+ )
225
+
226
+ config = get_config()
227
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
228
+ index 4068fc8..d284a23 100644
229
+ --- a/sample/llava/conversation.py
230
+ +++ b/sample/llava/conversation.py
231
+ @@ -99,7 +99,7 @@ class Conversation:
232
+ if self.version == "llama_v3":
233
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
234
+ elif self.version == "llava_llada":
235
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
236
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
237
+ else:
238
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
239
+ chat_template_messages = [{"role": "system", "content": self.system}]
240
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
241
+ index 3db5302..96e33b5 100644
242
+ --- a/sample/llava/model/builder.py
243
+ +++ b/sample/llava/model/builder.py
244
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
245
+ elif "llada" in model_name.lower():
246
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
247
+
248
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
249
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
250
+ +
251
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
252
+ if customized_config is None:
253
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
254
+ else:
255
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
256
+ index 4068fc8..d284a23 100644
257
+ --- a/train/llava/conversation.py
258
+ +++ b/train/llava/conversation.py
259
+ @@ -99,7 +99,7 @@ class Conversation:
260
+ if self.version == "llama_v3":
261
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
262
+ elif self.version == "llava_llada":
263
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
264
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
265
+ else:
266
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
267
+ chat_template_messages = [{"role": "system", "content": self.system}]
268
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
269
+ index 3db5302..aafab95 100644
270
+ --- a/train/llava/model/builder.py
271
+ +++ b/train/llava/model/builder.py
272
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
273
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
274
+ elif "llada" in model_name.lower():
275
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
276
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
277
+
278
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
279
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
280
+ if customized_config is None:
281
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
282
+ else:
283
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
284
+ index 3da28dc..8b17da3 100644
285
+ --- a/train/llava/model/language_model/llava_llada.py
286
+ +++ b/train/llava/model/language_model/llava_llada.py
287
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
288
+ dpo_forward: Optional[bool] = None,
289
+ cache_position=None,
290
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
291
+ -
292
+ + conversation_ids = None
293
+ if inputs_embeds is None and attention_mask is not None:
294
+ # donate multi-dialogue
295
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
296
+ diff --git a/train/rl_llada.py b/train/rl_llada.py
297
+ index 916ce23..00693be 100644
298
+ --- a/train/rl_llada.py
299
+ +++ b/train/rl_llada.py
300
+ @@ -96,7 +96,7 @@ def main():
301
+ accelerator = Accelerator(
302
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
303
+ mixed_precision=config.training.mixed_precision,
304
+ - log_with=None,
305
+ + log_with='wandb',
306
+ project_dir=config.experiment.logging_dir,
307
+ split_batches=True,
308
+ )
wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml
2
+ index 3e07fbd..337824b 100644
3
+ --- a/configs/lladav_eval.yaml
4
+ +++ b/configs/lladav_eval.yaml
5
+ @@ -7,17 +7,17 @@ system:
6
+ HTTP_PROXY: null
7
+ HF_HOME: null
8
+ env_name: ""
9
+ - envs_dir: "/abs/path/of/envs"
10
+ - base_dir: "/abs/path/of/dLLM-RL"
11
+ + envs_dir: null
12
+ + base_dir: "/data4/runqing/dLLM-RL"
13
+
14
+ -model: "/abs/path/of/model"
15
+ +model: "/data4/runqing/dLLM-RL/LLaDA-V"
16
+ model_base: "lladav"
17
+ answer_must_in_box: True
18
+
19
+ dataset:
20
+ eval_dataset: "SEEDBench_IMG_32"
21
+ data_type: "mmu"
22
+ - image_root: "/abs/path/of/data/images"
23
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
24
+
25
+ execute:
26
+ num_chunk: 128
27
+ @@ -35,4 +35,3 @@ rollout:
28
+ further_horizon: 128
29
+ output_unmasking_history: True
30
+ use_cache: True
31
+ -
32
+ diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml
33
+ index 7e38050..a07f620 100644
34
+ --- a/configs/rl_lladav.yaml
35
+ +++ b/configs/rl_lladav.yaml
36
+ @@ -5,9 +5,9 @@ wandb:
37
+ system:
38
+ HTTP_PROXY: null
39
+ HF_HOME: null
40
+ - env_name: "env_name"
41
+ - envs_dir: "/abs/path/of/envs"
42
+ - base_dir: "/abs/path/of/dLLM-RL"
43
+ + env_name: "/data8/runqing/miniconda/envs/dllm-rl"
44
+ + envs_dir: "/data8/runqing/miniconda/envs/dllm-rl"
45
+ + base_dir: "/data4/runqing/dLLM-RL"
46
+
47
+ experiment:
48
+ project: "rl_lladav"
49
+ @@ -22,17 +22,17 @@ experiment:
50
+ node_index: 0
51
+
52
+ model:
53
+ - pretrained_model: "/abs/path/of/sft-ckpt"
54
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
55
+ optimized_name: "optimized"
56
+ model_base: "lladav"
57
+ - vq_model_path: "/abs/path/of/magvitv2"
58
+ + vq_model_path: "/data4/runqing/hf/magvitv2"
59
+ image_resolution: 512
60
+
61
+ dataset:
62
+ train_dataset: "SEEDBench_IMG_32"
63
+ optimization_data: "rl_data"
64
+ data_type: "mmu"
65
+ - image_root: "/abs/path/of/data/images"
66
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
67
+
68
+ rollout:
69
+ num_task_per_step: 128
70
+ diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml
71
+ index 6557210..001a6e2 100644
72
+ --- a/configs/sft_lladav.yaml
73
+ +++ b/configs/sft_lladav.yaml
74
+ @@ -7,13 +7,13 @@ experiment:
75
+ num_node: 1
76
+
77
+ model:
78
+ - pretrained_model: "/abs/path/of/model"
79
+ + pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V"
80
+ optimized_name: "optimized"
81
+
82
+ dataset:
83
+ - optimization_data: "sft_test" # ./data/sft_test.json
84
+ + optimization_data: "sft_test_v" # ./data/sft_test.json
85
+ data_type: "mmu"
86
+ - image_root: "/base_dir/data/images"
87
+ + image_root: "/data4/runqing/dLLM-RL/data/images"
88
+
89
+ training:
90
+ gradient_checkpointing_enable: False
91
+ diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb
92
+ index e0b183f..9f94fa7 100644
93
+ --- a/data/preprocess_sft.ipynb
94
+ +++ b/data/preprocess_sft.ipynb
95
+ @@ -30,15 +30,15 @@
96
+ "#eos_token = \"<|im_end|>\"\n",
97
+ "\n",
98
+ "# llada and mmada:\n",
99
+ - "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
100
+ - "#eos_token = \"<|eot_id|>\"\n",
101
+ + "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n",
102
+ + "eos_token = \"<|eot_id|>\"\n",
103
+ "\n",
104
+ "# trado and sdar:\n",
105
+ "# non-cot prompt, we used this as demon example to compare sft methods\n",
106
+ - "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
107
+ + "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n",
108
+ "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n",
109
+ "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n",
110
+ - "eos_token = \"<|im_end|>\"\n",
111
+ + "# eos_token = \"<|im_end|>\"\n",
112
+ "\n",
113
+ "\n",
114
+ "def get_prompt(data_i):\n",
115
+ @@ -67,6 +67,87 @@
116
+ "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n",
117
+ " json.dump(processed_data, f, indent=2, ensure_ascii=False)"
118
+ ]
119
+ + },
120
+ + {
121
+ + "cell_type": "code",
122
+ + "execution_count": 2,
123
+ + "id": "32bbd872",
124
+ + "metadata": {},
125
+ + "outputs": [
126
+ + {
127
+ + "name": "stdout",
128
+ + "output_type": "stream",
129
+ + "text": [
130
+ + "100\n"
131
+ + ]
132
+ + }
133
+ + ],
134
+ + "source": [
135
+ + "import json\n",
136
+ + "\n",
137
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n",
138
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n",
139
+ + "save_data_list = []\n",
140
+ + "save_num = 0\n",
141
+ + "with open(data_path, 'r') as fin, \\\n",
142
+ + " open(save_path, 'w') as fout:\n",
143
+ + " raw_data = json.load(fin)\n",
144
+ + " for data in raw_data:\n",
145
+ + " if save_num < 100:\n",
146
+ + " save_data_list.append(data)\n",
147
+ + " save_num += 1\n",
148
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
149
+ + " print(len(save_data_list))\n"
150
+ + ]
151
+ + },
152
+ + {
153
+ + "cell_type": "code",
154
+ + "execution_count": null,
155
+ + "id": "fdfde30f",
156
+ + "metadata": {},
157
+ + "outputs": [
158
+ + {
159
+ + "name": "stdout",
160
+ + "output_type": "stream",
161
+ + "text": [
162
+ + "200\n"
163
+ + ]
164
+ + },
165
+ + {
166
+ + "ename": "",
167
+ + "evalue": "",
168
+ + "output_type": "error",
169
+ + "traceback": [
170
+ + "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
171
+ + "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n",
172
+ + "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
173
+ + "\u001b[1;31m有关更多详细信息,请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
174
+ + ]
175
+ + }
176
+ + ],
177
+ + "source": [
178
+ + "import json\n",
179
+ + "import random\n",
180
+ + "random.seed(2026)\n",
181
+ + "\n",
182
+ + "\n",
183
+ + "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n",
184
+ + "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n",
185
+ + "save_data_list = []\n",
186
+ + "save_num = 0\n",
187
+ + "with open(data_path, 'r') as fin, \\\n",
188
+ + " open(save_path, 'w') as fout:\n",
189
+ + " raw_data = json.load(fin)\n",
190
+ + "\n",
191
+ + " # random.shuffle(raw_data)\n",
192
+ + " for did, data in enumerate(raw_data):\n",
193
+ + " # if did > 1999:\n",
194
+ + " if save_num < 200:\n",
195
+ + " save_data_list.append(data)\n",
196
+ + " save_num += 1\n",
197
+ + " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n",
198
+ + " print(len(save_data_list))\n"
199
+ + ]
200
+ }
201
+ ],
202
+ "metadata": {
203
+ @@ -85,7 +166,7 @@
204
+ "name": "python",
205
+ "nbconvert_exporter": "python",
206
+ "pygments_lexer": "ipython3",
207
+ - "version": "3.10.18"
208
+ + "version": "3.10.19"
209
+ }
210
+ },
211
+ "nbformat": 4,
212
+ diff --git a/rl_v.py b/rl_v.py
213
+ index 8f2c0f1..79d2e5b 100644
214
+ --- a/rl_v.py
215
+ +++ b/rl_v.py
216
+ @@ -15,7 +15,9 @@ if __name__ == "__main__":
217
+ def env_prefix(env) -> str:
218
+ return (
219
+ "source ~/.bashrc && "
220
+ - f"source activate {env} && "
221
+ + "source /data8/runqing/miniconda/bin/activate && "
222
+ + "conda init && "
223
+ + f"conda activate {env} && "
224
+ )
225
+
226
+ config = get_config()
227
+ diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py
228
+ index 4068fc8..d284a23 100644
229
+ --- a/sample/llava/conversation.py
230
+ +++ b/sample/llava/conversation.py
231
+ @@ -99,7 +99,7 @@ class Conversation:
232
+ if self.version == "llama_v3":
233
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
234
+ elif self.version == "llava_llada":
235
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
236
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
237
+ else:
238
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
239
+ chat_template_messages = [{"role": "system", "content": self.system}]
240
+ diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py
241
+ index 3db5302..96e33b5 100644
242
+ --- a/sample/llava/model/builder.py
243
+ +++ b/sample/llava/model/builder.py
244
+ @@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
245
+ elif "llada" in model_name.lower():
246
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
247
+
248
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
249
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
250
+ +
251
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
252
+ if customized_config is None:
253
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
254
+ else:
255
+ diff --git a/train/llava/conversation.py b/train/llava/conversation.py
256
+ index 4068fc8..d284a23 100644
257
+ --- a/train/llava/conversation.py
258
+ +++ b/train/llava/conversation.py
259
+ @@ -99,7 +99,7 @@ class Conversation:
260
+ if self.version == "llama_v3":
261
+ self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True)
262
+ elif self.version == "llava_llada":
263
+ - self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True)
264
+ + self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True)
265
+ else:
266
+ raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.")
267
+ chat_template_messages = [{"role": "system", "content": self.system}]
268
+ diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py
269
+ index 3db5302..aafab95 100644
270
+ --- a/train/llava/model/builder.py
271
+ +++ b/train/llava/model/builder.py
272
+ @@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
273
+ model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs)
274
+ elif "llada" in model_name.lower():
275
+ from llava.model.language_model.llava_llada import LlavaLLaDAConfig
276
+ + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
277
+
278
+ - tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
279
+ + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
280
+ if customized_config is None:
281
+ llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path)
282
+ else:
283
+ diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py
284
+ index 3da28dc..8b17da3 100644
285
+ --- a/train/llava/model/language_model/llava_llada.py
286
+ +++ b/train/llava/model/language_model/llava_llada.py
287
+ @@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM):
288
+ dpo_forward: Optional[bool] = None,
289
+ cache_position=None,
290
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
291
+ -
292
+ + conversation_ids = None
293
+ if inputs_embeds is None and attention_mask is not None:
294
+ # donate multi-dialogue
295
+ (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True)
296
+ diff --git a/train/rl_llada.py b/train/rl_llada.py
297
+ index 916ce23..00693be 100644
298
+ --- a/train/rl_llada.py
299
+ +++ b/train/rl_llada.py
300
+ @@ -96,7 +96,7 @@ def main():
301
+ accelerator = Accelerator(
302
+ gradient_accumulation_steps=config.training.gradient_accumulation_steps,
303
+ mixed_precision=config.training.mixed_precision,
304
+ - log_with=None,
305
+ + log_with='wandb',
306
+ project_dir=config.experiment.logging_dir,
307
+ split_batches=True,
308
+ )
wandb/run-20260325_141344-o1lsxhwu/files/output.log ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 03/25/2026 14:13:47 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml
2
+ 03/25/2026 14:13:47 - INFO - __main__ - Loading LLaDA-V model and tokenizer
3
+ Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal/ckpt/optimized
4
+ Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384
5
+ Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.72s/it]
6
+ Rank 0: Model Class: LlavaLLaDAModelLM
7
+ 03/25/2026 14:14:14 - INFO - __main__ - Preprocessing Data
8
+ 03/25/2026 14:14:14 - INFO - __main__ - Preparing model, optimizer and dataloaders
9
+ /data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
10
+ If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
11
+ warnings.warn(
12
+ Before initializing optimizer states
13
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
14
+ CPU Virtual Memory: used = 181.57 GB, percent = 9.0%
15
+ After initializing optimizer states
16
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
17
+ CPU Virtual Memory: used = 194.7 GB, percent = 9.7%
18
+ After initializing ZeRO optimizer
19
+ MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB
20
+ CPU Virtual Memory: used = 194.31 GB, percent = 9.6%
21
+ 03/25/2026 14:15:29 - INFO - __main__ - ***** Running inference *****
22
+ 03/25/2026 14:15:29 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants...
23
+ 03/25/2026 14:19:13 - INFO - __main__ - ***** Running training *****
24
+ 03/25/2026 14:19:13 - INFO - __main__ - Num variants = 560
25
+ 03/25/2026 14:19:13 - INFO - __main__ - Num training steps = 36
26
+ 03/25/2026 14:19:13 - INFO - __main__ - Instantaneous batch size per device = 1
27
+ 03/25/2026 14:19:13 - INFO - __main__ - Total train batch size (parallel * accum) = 16
28
+ 03/25/2026 14:19:13 - INFO - __main__ - Gradient Accumulation steps = 2
29
+ start rl training...
30
+ Epoch 1/1: 100%|██████████| 70/70 [18:59<00:00, 16.28s/it]
31
+ 03/25/2026 14:38:51 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal/ckpt/optimized
wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ transformers==4.52.4
2
+ nvidia-nvtx-cu12==12.4.127
3
+ deepspeed==0.18.6
4
+ typing-inspection==0.4.2
5
+ scipy==1.15.3
6
+ MarkupSafe==3.0.3
7
+ nvidia-cuda-runtime-cu12==12.4.127
8
+ yarl==1.22.0
9
+ tornado==6.5.4
10
+ nvidia-curand-cu12==10.3.5.147
11
+ nest-asyncio==1.6.0
12
+ safetensors==0.7.0
13
+ async-timeout==5.0.1
14
+ typing_extensions==4.15.0
15
+ asttokens==3.0.1
16
+ cuda-python==13.1.1
17
+ executing==2.2.1
18
+ nvidia-cudnn-frontend==1.18.0
19
+ comm==0.2.3
20
+ tokenizers==0.21.4
21
+ six==1.17.0
22
+ Pygments==2.19.2
23
+ nvidia-cusparselt-cu12==0.6.2
24
+ apache-tvm-ffi==0.1.8.post2
25
+ flash_attn==2.7.4.post1
26
+ tqdm==4.67.3
27
+ tensorboard-data-server==0.7.2
28
+ annotated-types==0.7.0
29
+ math-verify==0.9.0
30
+ idna==3.11
31
+ regex==2026.1.15
32
+ nvidia-cutlass-dsl-libs-base==4.4.0
33
+ flashinfer-python==0.6.3
34
+ click==8.3.1
35
+ pytz==2025.2
36
+ wheel==0.46.3
37
+ ipykernel==7.2.0
38
+ stack-data==0.6.3
39
+ absl-py==2.4.0
40
+ wandb==0.25.0
41
+ pandas==2.3.3
42
+ termcolor==3.3.0
43
+ sympy==1.13.1
44
+ joblib==1.5.3
45
+ prompt_toolkit==3.0.52
46
+ attrs==25.4.0
47
+ nvidia-cusolver-cu12==11.6.1.9
48
+ xxhash==3.6.0
49
+ aiohappyeyeballs==2.6.1
50
+ python-dateutil==2.9.0.post0
51
+ frozenlist==1.8.0
52
+ nvidia-ml-py==13.590.48
53
+ multiprocess==0.70.18
54
+ decorator==5.2.1
55
+ cuda-pathfinder==1.3.4
56
+ multidict==6.7.1
57
+ pure_eval==0.2.3
58
+ nvidia-cufft-cu12==11.2.1.3
59
+ omegaconf==2.3.0
60
+ PyYAML==6.0.3
61
+ debugpy==1.8.20
62
+ psutil==7.2.2
63
+ ninja==1.13.0
64
+ nvidia-cublas-cu12==12.4.5.8
65
+ triton==3.2.0
66
+ antlr4-python3-runtime==4.9.3
67
+ peft==0.18.1
68
+ aiosignal==1.4.0
69
+ accelerate==1.12.0
70
+ liger_kernel==0.7.0
71
+ anyio==4.12.1
72
+ decord==0.6.0
73
+ h11==0.16.0
74
+ urllib3==2.6.3
75
+ wcwidth==0.6.0
76
+ networkx==3.4.2
77
+ huggingface_hub==0.36.2
78
+ tzdata==2025.3
79
+ xformers==0.0.29.post3
80
+ tabulate==0.9.0
81
+ certifi==2026.1.4
82
+ tensorboard==2.20.0
83
+ matplotlib-inline==0.2.1
84
+ pip==26.0.1
85
+ smmap==5.0.2
86
+ nvidia-cuda-cupti-cu12==12.4.127
87
+ Markdown==3.10.2
88
+ nvidia-nccl-cu12==2.21.5
89
+ aiohttp==3.13.3
90
+ scikit-learn==1.7.2
91
+ nvidia-cutlass-dsl==4.4.0
92
+ setuptools==82.0.0
93
+ jupyter_core==5.9.1
94
+ asyncio==4.0.0
95
+ einops==0.8.2
96
+ pydantic==2.12.5
97
+ av==16.1.0
98
+ protobuf==6.33.5
99
+ evaluate==0.4.6
100
+ Jinja2==3.1.6
101
+ exceptiongroup==1.3.1
102
+ ipython==8.38.0
103
+ sentry-sdk==2.53.0
104
+ httpx==0.28.1
105
+ msgpack==1.1.2
106
+ platformdirs==4.9.2
107
+ hjson==3.1.0
108
+ torchvision==0.21.0
109
+ parso==0.8.6
110
+ cuda-bindings==13.1.1
111
+ pexpect==4.9.0
112
+ threadpoolctl==3.6.0
113
+ pyarrow==23.0.1
114
+ jedi==0.19.2
115
+ nvidia-cuda-nvrtc-cu12==12.4.127
116
+ Werkzeug==3.1.5
117
+ fsspec==2025.10.0
118
+ pydantic_core==2.41.5
119
+ GitPython==3.1.46
120
+ propcache==0.4.1
121
+ mpmath==1.3.0
122
+ nvidia-nvjitlink-cu12==12.4.127
123
+ nvidia-cusparse-cu12==12.3.1.170
124
+ hf-xet==1.2.0
125
+ charset-normalizer==3.4.4
126
+ dill==0.4.0
127
+ datasets==4.5.0
128
+ py-cpuinfo==9.0.0
129
+ sentencepiece==0.2.1
130
+ latex2sympy2_extended==1.11.0
131
+ jupyter_client==8.8.0
132
+ pyzmq==27.1.0
133
+ numpy==2.2.6
134
+ gitdb==4.0.12
135
+ grpcio==1.78.0
136
+ packaging==26.0
137
+ requests==2.32.5
138
+ nvidia-cudnn-cu12==9.1.0.70
139
+ httpcore==1.0.9
140
+ traitlets==5.14.3
141
+ filelock==3.24.2
142
+ ptyprocess==0.7.0
143
+ pillow==12.1.1
144
+ torch==2.6.0
145
+ more-itertools==10.8.0
146
+ wheel==0.46.3
147
+ jaraco.text==4.0.0
148
+ zipp==3.23.0
149
+ autocommand==2.2.2
150
+ jaraco.context==6.1.0
151
+ jaraco.functools==4.4.0
152
+ tomli==2.4.0
153
+ backports.tarfile==1.2.0
154
+ importlib_metadata==8.7.1
155
+ platformdirs==4.4.0
156
+ packaging==26.0
wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.19",
4
+ "startedAt": "2026-03-25T06:13:44.186076Z",
5
+ "args": [
6
+ "config=configs/rl_lladav_video_reversal.yaml",
7
+ "experiment.current_epoch=27"
8
+ ],
9
+ "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py",
10
+ "codePath": "train/rl_lladav.py",
11
+ "codePathLocal": "train/rl_lladav.py",
12
+ "git": {
13
+ "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
14
+ "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
15
+ },
16
+ "email": "shixi_runqing@staff.weibo.com",
17
+ "root": "/data4/runqing/dLLM-RL",
18
+ "host": "b1301b38c75f",
19
+ "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H800",
23
+ "gpu_count": 8,
24
+ "disk": {
25
+ "/": {
26
+ "total": "806578724864",
27
+ "used": "572884140032"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "2164113813504"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H800",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
40
+ },
41
+ {
42
+ "name": "NVIDIA H800",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
47
+ },
48
+ {
49
+ "name": "NVIDIA H800",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
54
+ },
55
+ {
56
+ "name": "NVIDIA H800",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
61
+ },
62
+ {
63
+ "name": "NVIDIA H800",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
68
+ },
69
+ {
70
+ "name": "NVIDIA H800",
71
+ "memoryTotal": "85520809984",
72
+ "cudaCores": 16896,
73
+ "architecture": "Hopper",
74
+ "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
75
+ },
76
+ {
77
+ "name": "NVIDIA H800",
78
+ "memoryTotal": "85520809984",
79
+ "cudaCores": 16896,
80
+ "architecture": "Hopper",
81
+ "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379"
82
+ },
83
+ {
84
+ "name": "NVIDIA H800",
85
+ "memoryTotal": "85520809984",
86
+ "cudaCores": 16896,
87
+ "architecture": "Hopper",
88
+ "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
89
+ }
90
+ ],
91
+ "cudaVersion": "12.4",
92
+ "writerId": "l1zvo1vvzkyq01b0wblads1a5x42yuiv"
93
+ }
wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_timestamp":1.774420691877827e+09,"_wandb":{"runtime":1504},"_runtime":1504.468829676,"_step":35,"train/loss":-1.004162073135376,"train/learning_rate":5.868240888334637e-07}